author Gregory Szorc <gps@mozilla.com>
Wed, 19 Sep 2012 11:20:19 -0700
changeset 107532 d5720a3513ca42ffd136e9f5a5f1d9b4ecebcc60
parent 94475 f4157e8c410708d76703f19e4dfb61859bfe32d8
child 142438 d5a9b3ef17063aeeacfe8dd08a2eb90f028dfd8a
permissions -rw-r--r--
Bug 783727 - Part 5: Add psutil to virtualenv; r=ted It is optional so if your machine does not support building Python C extensions things should still work.

#! /usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Script name:   talos_from_code.py
# Purpose:       Read from a talos.json file the different files to download for a talos job
# Author(s):     Zambrano Gasparnian, Armen <armenzg@mozilla.com>
# Target:        Python 2.5
from optparse import OptionParser
    import json
    import simplejson as json
import re
import urllib2
import urlparse
import sys
import os

def main():
    This script downloads a talos.json file which indicates which files to download
    for a talos job.
    See a talos.json file for a better understand:
    parser = OptionParser()
    parser.add_option("--talos-json-url", dest="talos_json_url", type="string",
                      help="It indicates from where to download the talos.json file.")
    (options, args) = parser.parse_args()

    # 1) check that the url was passed
    if options.talos_json_url == None:
        print "You need to specify --talos-json-url."

    # 2) try to download the talos.json file
        jsonFilename = download_file(options.talos_json_url)
    except Exception, e:
        print "ERROR: We tried to download the talos.json file but something failed."
        print "ERROR: %s" % str(e)

    # 3) download the necessary files
    print "INFO: talos.json URL: %s" % options.talos_json_url
        key = 'talos.zip'
        entity = get_value(jsonFilename, key)
        if passesRestrictions(options.talos_json_url, entity["url"]):
            # the key is at the same time the filename e.g. talos.zip
            print "INFO: Downloading %s as %s" % (entity["url"], os.path.join(entity["path"], key))
            download_file(entity["url"], entity["path"], key)
            print "ERROR: You have tried to download a file " + \
                  "from: %s " % fileUrl + \
                  "which is a location different than http://build.mozilla.org/talos/"
            print "ERROR: This is only allowed for the certain branches."
    except Exception, e:
        print "ERROR: %s" % str(e)

def passesRestrictions(talosJsonUrl, fileUrl):
    Only certain branches are exempted from having to host their downloadable files
    in build.mozilla.org
    if talosJsonUrl.startswith("http://hg.mozilla.org/try/") == True or \
       talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") == True:
        return True
        p = re.compile('^http://build.mozilla.org/talos/')
        m = p.match(fileUrl)
        if m == None:
            return False
        return True

def get_filename_from_url(url):
    This returns the filename of the file we're trying to download
    parsed = urlparse.urlsplit(url.rstrip('/'))
    if parsed.path != '':
        return parsed.path.rsplit('/', 1)[-1]
        print "ERROR: We were trying to download a file from %s " + \
              "but the URL seems to be incorrect."

def download_file(url, path="", saveAs=None):
    It downloads a file from URL to the indicated path
    req = urllib2.Request(url)
    f = urllib2.urlopen(req)
    if path != "" and not os.path.isdir(path):
            print "INFO: directory %s created" % path
        except Exception, e:
            print "ERROR: %s" % str(e)
    filename = saveAs if saveAs else get_filename_from_url(url)
    local_file = open(os.path.join(path, filename), 'wb')
    return filename

def get_value(json_filename, key):
    It loads up a JSON file and returns the value for the given string
    f = open(json_filename, 'r')
    return json.load(f)[key]

if __name__ == '__main__':