author Brian O'Keefe <>
Tue, 17 Mar 2015 12:53:48 -0400
changeset 29213 201e66c2d6a7ed6f3f4fcbf1dddabbaddffde349
parent 27970 74916ecc3d77bfd9520338828d22cd7afaa810c6
child 29866 47eaee3825cadf1d4012fbb4b63c6ea0cc8e55f1
permissions -rwxr-xr-x
Bug 1138063 - Clean up a bunch of make rules that install things to FINAL_TARGET; r=jcranmer,fallen,florian

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at
import argparse, ftplib, json, os, os.path, re, shutil, subprocess, sys, tarfile, tempfile
from collections import OrderedDict

def parse_args():
    """Gather arguments from the command-line."""
    parser = argparse.ArgumentParser(description="Create timezone info JSON file from tzdata files")
    parser.add_argument("-v", "--vzic", dest="vzic_path", required=True,
        help="""Path to the `vzic` executable. This must be downloaded
        from and compiled."""
    parser.add_argument("-t", "--tzdata", dest="tzdata_path",
        help="""Path to a directory containing the IANA timezone data.
        If this argument is omitted, the data will be downloaded from"""
    return parser.parse_args()

def prepare_tzdata():
    """Download timezone info, or use a local copy, if specified.
    Then use vzic to create ICS versions of the data."""
    args = parse_args()

    if args.tzdata_path is None:
        tzdata_download_path = tempfile.mktemp(".tar.gz", prefix="zones")
        sys.stderr.write("Downloading tzdata-latest.tar.gz from to %s\n" % tzdata_download_path)
        ftp = ftplib.FTP("")
        ftp.retrbinary("RETR /tz/tzdata-latest.tar.gz", open(tzdata_download_path, "wb").write)

        tzdata_path = tempfile.mkdtemp(prefix="zones")
        sys.stderr.write("Extracting %s to %s\n" % (tzdata_download_path, tzdata_path))
        tzdata_path = args.tzdata_path

    # Extract version number of tzdata files.
    with open(os.path.join(tzdata_path, "Makefile"), "r") as fp:
        for line in fp:
            match ="VERSION=\s*(\w+)", line)
            if match is not None:
                version = "2." +

    # Use `vzic` to create 'pure' and 'non-pure' zone files.
    sys.stderr.write("Exporting zone info to %s\n" % zoneinfo_path)
        "--olson-dir", tzdata_path,
        "--output-dir", zoneinfo_path
    ], stdout=sys.stderr)

    sys.stderr.write("Exporting pure zone info to %s\n" % zoneinfo_pure_path)
        "--olson-dir", tzdata_path,
        "--output-dir", zoneinfo_pure_path,
    ], stdout=sys.stderr)

    if args.tzdata_path is None:

    return version

def read_zones_tab():
    """Read for latitude and longitude data."""
    lat_long_data = {}
    with open(os.path.join(zoneinfo_path, ""), "r") as tab:
        for line in tab:
            if len(line) < 19:
                sys.stderr.write("Line in not long enough: %s\n" % line.strip())

            [latitude, longitude, name] = line.rstrip().split(" ", 2)
            lat_long_data[name] = (latitude, longitude)
    return lat_long_data

def read_ics(filename):
    """Read a single zone's ICS files.

    We keep only the lines we want, and we use the pure version of RRULE if
    the versions differ. See Asia/Jerusalem for an example."""
    with open(os.path.join(zoneinfo_path, filename), "r") as fp:
        zoneinfo = fp.readlines()

    with open(os.path.join(zoneinfo_pure_path, filename), "r") as fp:
        zoneinfo_pure = fp.readlines()

    ics_data = []
    for i in range(0, len(zoneinfo)):
        line = zoneinfo[i]
        key = line[:line.find(":")]

        if key == "BEGIN":
            if line != "BEGIN:VCALENDAR\r\n":
        elif key == "END":
            if line != "END:VCALENDAR\r\n":
        elif key in ("TZID", "TZOFFSETFROM", "TZOFFSETTO", "TZNAME", "DTSTART"):
        elif key == "RRULE":
            if line == zoneinfo_pure[i]:
                sys.stderr.write("Using pure version of %s\n" % filename[:-4])

    zone_data = {
        "ics": "".join(ics_data).rstrip()
    zone_name = filename[:-4]
    if zone_name in lat_long_data:
        zone_data["latitude"] = lat_long_data[zone_name][0]
        zone_data["longitude"] = lat_long_data[zone_name][1]

    return zone_data

def read_dir(path, prefix=""):
    """Recursively read a directory for ICS files.

    Files could be two or three levels deep."""
    zones = {}
    for entry in os.listdir(path):
        fullpath = os.path.join(path, entry)
        if os.path.isdir(fullpath):
            zones.update(read_dir(fullpath, os.path.join(prefix, entry)))
        elif prefix != "":
            filename = os.path.join(prefix, entry)
            zones[filename[:-4]] = read_ics(filename)
    return zones

def read_aliases():
    """Copy the list of aliases from the previous version of zones.json."""
    with open(json_file, "r") as fp:
        json_data = json.load(fp)
        return json_data["aliases"]

def write_output(version, aliases, zones):
    """Write the data to zones.json."""
    data = OrderedDict()
    data["version"] = version
    data["aliases"] = OrderedDict(sorted(aliases.items()))
    data["zones"] = OrderedDict(sorted(zones.items()))

    with open(json_file, "w") as fp:
        json.dump(data, fp, indent=2, separators=(",", ": "))

json_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "zones.json")
zoneinfo_path = tempfile.mkdtemp(prefix="zones")
zoneinfo_pure_path = tempfile.mkdtemp(prefix="zones")

version = prepare_tzdata()
lat_long_data = read_zones_tab()
zones = read_dir(zoneinfo_path)
aliases = read_aliases()

write_output(version, aliases, zones)

# Clean up.