Bug 642411: teach setup-master.py and friends to read masters json. r=bhearsum
authorChris AtLee <catlee@mozilla.com>
Wed, 18 May 2011 21:37:06 -0400
changeset 4120 93313771f55e27a8971135ff3c8081aaae165266
parent 4119 8cd625e493dfb1f8e7faf52e600a6c0fb8409d2b
child 4121 7471fcffcbc356371f0cef20c85b815efc2170dd
child 4123 870b24a4b96169827c2fada3f7695e9b6573cc27
push id1
push userroot
push dateWed, 17 Dec 2014 00:18:48 +0000
reviewersbhearsum
bugs642411
Bug 642411: teach setup-master.py and friends to read masters json. r=bhearsum also make -8 the default, and add a -7 option for testing/setting up new masters
Makefile.setup
mozilla-tests/tests_localconfig.py
mozilla/build_localconfig.py
mozilla/try_localconfig.py
setup-master.py
test-masters.sh
--- a/Makefile.setup
+++ b/Makefile.setup
@@ -1,29 +1,35 @@
 BASEDIR ?= $(shell pwd)
 VIRTUALENV ?= virtualenv
 PYTHON ?= /tools/python/bin/python
 PIP_PACKAGES_FILE ?= master-pip.txt
 HG ?= hg
 HG_HOST = http://hg.mozilla.org
 HG_DIR = build
-MASTER_NAME = $(error MASTER_NAME must be defined (see setup-master.py -8 -l for a list))
+MASTER_NAME = $(error MASTER_NAME must be defined (see setup-master.py -l for a list))
 DATABASE = tm-b01-master01.mozilla.org/buildbot
 MYSQL_CONFIG = $(HOME)/.$(MASTER_NAME)_update_from_files.conf
 CRONTAB = /etc/cron.d/$(MASTER_NAME)
 VIRTUALENV_PYTHON = $(BASEDIR)/bin/python
 PIP = $(BASEDIR)/bin/pip
 BUILDBOT = $(BASEDIR)/bin/buildbot
 USER = cltbld
 HOME = /home/$(USER)
 BUILDBOT_BRANCH = production-0.8
 BUILDBOTCUSTOM_BRANCH = production-0.8
 BUILDBOTCONFIGS_BRANCH = production
 INSTALL_BUILDBOT_SLAVE=
 
+ifdef MASTERS_JSON
+SETUP_MASTER=setup-master.py -j $(MASTERS_JSON)
+else
+SETUP_MASTER=setup-master.py
+endif
+
 all: virtualenv deps install-buildbot master master-makefile other-files done
 
 done:
 	@echo
 	@echo
 	@echo You probably want to update at least the following files to contain accurate passwords/descriptions:
 	@echo BuildSlaves.py, passwords.py, $(MYSQL_CONFIG)
 	@echo You should also run the "cron" target as root, with the same variables set.
@@ -33,30 +39,30 @@ virtualenv:
 
 deps:
 	$(PIP) install $(PIP_FLAGS) -r $(PIP_PACKAGES_FILE)
 	
 # HACK ALERT: This only works when there's only one python version in "lib".
 # This is generally true when using virtualenvs.
 install-buildbot: SITE_PACKAGES = $(BASEDIR)/lib/$(shell ls $(BASEDIR)/lib/)/site-packages
 install-buildbot:
-	$(HG) clone -b $(BUILDBOT_BRANCH) $(HG_HOST)/$(HG_DIR)/buildbot $(BASEDIR)/buildbot
+	test -d $(BASEDIR)/buildbot || $(HG) clone -b $(BUILDBOT_BRANCH) $(HG_HOST)/$(HG_DIR)/buildbot $(BASEDIR)/buildbot
 	(cd $(BASEDIR)/buildbot/master; $(VIRTUALENV_PYTHON) setup.py develop install)
 ifdef INSTALL_BUILDBOT_SLAVE
 	(cd $(BASEDIR)/buildbot/slave; $(VIRTUALENV_PYTHON) setup.py develop install)
 endif
-	$(HG) clone -b $(BUILDBOTCUSTOM_BRANCH) $(HG_HOST)/$(HG_DIR)/buildbotcustom $(BASEDIR)/buildbotcustom
-	$(HG) clone -b $(BUILDBOTCONFIGS_BRANCH) $(HG_HOST)/$(HG_DIR)/buildbot-configs $(BASEDIR)/buildbot-configs
-	$(HG) clone $(HG_HOST)/$(HG_DIR)/tools $(BASEDIR)/tools
+	test -d $(BASEDIR)/buildbotcustom || $(HG) clone -b $(BUILDBOTCUSTOM_BRANCH) $(HG_HOST)/$(HG_DIR)/buildbotcustom $(BASEDIR)/buildbotcustom
+	test -d $(BASEDIR)/buildbot-configs || $(HG) clone -b $(BUILDBOTCONFIGS_BRANCH) $(HG_HOST)/$(HG_DIR)/buildbot-configs $(BASEDIR)/buildbot-configs
+	test -d $(BASEDIR)/tools || $(HG) clone $(HG_HOST)/$(HG_DIR)/tools $(BASEDIR)/tools
 	# Get buildbotcustom and the build/tools library into PYTHONPATH
-	ln -s $(BASEDIR)/buildbotcustom $(SITE_PACKAGES)/buildbotcustom
+	ln -sf $(BASEDIR)/buildbotcustom $(SITE_PACKAGES)/buildbotcustom
 	echo "$(BASEDIR)/tools/lib/python" > $(SITE_PACKAGES)/build-tools-lib.pth
 
 master:
-	(cd $(BASEDIR)/buildbot-configs; $(VIRTUALENV_PYTHON) setup-master.py -8 -b $(BUILDBOT) $(BASEDIR)/master $(MASTER_NAME))
+	(cd $(BASEDIR)/buildbot-configs; $(VIRTUALENV_PYTHON) $(SETUP_MASTER) -b $(BUILDBOT) $(BASEDIR)/master $(MASTER_NAME))
 
 master-makefile:
 	cp $(BASEDIR)/buildbot-configs/Makefile.master $(BASEDIR)/Makefile
 
 other-files:
 	echo "[DEFAULT]" > $(MYSQL_CONFIG)
 	echo "database = mysql://buildbot:password@$(DATABASE)" >> $(MYSQL_CONFIG)
 	echo "master = http://this-master:port" >> $(MYSQL_CONFIG)
new file mode 100644
--- /dev/null
+++ b/mozilla-tests/tests_localconfig.py
@@ -0,0 +1,25 @@
+from buildbot.util import json
+from buildbot.status.html import WebStatus
+from buildbot import manhole
+
+master_config = json.load(open('master_config.json'))
+
+c = BuildmasterConfig = {}
+c['slavePortnum'] = master_config.get('pb_port', 0)
+c['status'] = []
+
+if 'http_port' in master_config:
+    c['status'].append(
+            WebStatus(http_port=master_config['http_port'], allowForce=True))
+    c['buildbotURL'] = 'http://%(hostname)s:%(http_port)i/' % master_config
+
+if 'ssh_port' in master_config:
+    c['manhole'] = manhole.PasswordManhole(
+            "tcp:%(ssh_port)i:interface=127.0.0.1" % master_config,
+            "cltbld", "password")
+
+from config import BRANCHES, PLATFORMS, PROJECTS
+# Do everything!
+ACTIVE_BRANCHES = BRANCHES.keys()
+ACTIVE_PLATFORMS = dict((k,None) for k in PLATFORMS.keys())
+ACTIVE_PROJECTS = PROJECTS.keys()
new file mode 100644
--- /dev/null
+++ b/mozilla/build_localconfig.py
@@ -0,0 +1,43 @@
+from buildbot.util import json
+from buildbot.status.html import WebStatus
+from buildbot import manhole
+
+master_config = json.load(open('master_config.json'))
+
+c = BuildmasterConfig = {}
+c['slavePortnum'] = master_config.get('pb_port', 0)
+c['status'] = []
+
+if 'http_port' in master_config:
+    c['status'].append(
+            WebStatus(http_port=master_config['http_port'], allowForce=True))
+    c['buildbotURL'] = 'http://%(hostname)s:%(http_port)i/' % master_config
+
+if 'ssh_port' in master_config:
+    c['manhole'] = manhole.PasswordManhole(
+            "tcp:%(ssh_port)i:interface=127.0.0.1" % master_config,
+            "cltbld", "password")
+
+from config import BRANCHES, SLAVES, PROJECTS, ACTIVE_PROJECT_BRANCHES
+ACTIVE_BRANCHES = ACTIVE_PROJECT_BRANCHES[:]
+ACTIVE_BRANCHES.extend([
+    'mozilla-1.9.1',
+    'mozilla-1.9.2',
+    'mozilla-2.0',
+    'mozilla-2.1',
+    'mozilla-central',
+    'shadow-central',
+    ])
+ACTIVE_PROJECTS = PROJECTS.keys()
+
+ACTIVE_RELEASE_BRANCHES = []
+
+# Set up our fast slaves
+# No need to reload, this is reloaded by builder_master.cfg
+import buildbotcustom.misc
+buildbotcustom.misc.fastRegexes.extend([
+    '-ix-',
+    'xserve',
+    ])
+ENABLE_RELEASES = False
+RESERVED_SLAVES = "reserved_slaves_%(name)s" % master_config
new file mode 100644
--- /dev/null
+++ b/mozilla/try_localconfig.py
@@ -0,0 +1,37 @@
+from buildbot.util import json
+from buildbot.status.html import WebStatus
+from buildbot import manhole
+
+master_config = json.load(open('master_config.json'))
+
+c = BuildmasterConfig = {}
+c['slavePortnum'] = master_config.get('pb_port', 0)
+c['status'] = []
+
+if 'http_port' in master_config:
+    c['status'].append(
+            WebStatus(http_port=master_config['http_port'], allowForce=True))
+    c['buildbotURL'] = 'http://%(hostname)s:%(http_port)i/' % master_config
+
+if 'ssh_port' in master_config:
+    c['manhole'] = manhole.PasswordManhole(
+            "tcp:%(ssh_port)i:interface=127.0.0.1" % master_config,
+            "cltbld", "password")
+
+from config import BRANCHES, TRY_SLAVES
+ACTIVE_BRANCHES = ['try']
+ACTIVE_RELEASE_BRANCHES = []
+ACTIVE_PROJECTS = []
+
+# Override with TRY_SLAVES
+SLAVES = TRY_SLAVES
+
+# Set up our fast slaves
+# No need to reload, this is reloaded by builder_master.cfg
+import buildbotcustom.misc
+buildbotcustom.misc.fastRegexes.extend([
+    '-ix-',
+    'xserve',
+    ])
+ENABLE_RELEASES = False
+RESERVED_SLAVES = None
--- a/setup-master.py
+++ b/setup-master.py
@@ -1,30 +1,36 @@
 #!/usr/bin/env python
 """setup-master.py master_dir master_name
 
 Sets up mozilla buildbot master in master_dir."""
 
-import os, glob, shutil, subprocess
+import os, glob, shutil, subprocess, urllib
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 class MasterConfig:
-    def __init__(self, name=None, config_dir=None, globs=None, renames=None, local_links=None):
+    def __init__(self, name=None, config_dir=None, globs=None, renames=None, local_links=None, extras=None):
         self.name = name or None
         self.config_dir = config_dir
         self.globs = globs or []
         self.renames = renames or []
         self.local_links = local_links or []
+        self.extras = extras or []
 
     def __add__(self, o):
         retval = MasterConfig(
                 name = self.name or o.name,
                 config_dir = self.config_dir or o.config_dir,
                 globs = self.globs + o.globs,
                 renames = self.renames + o.renames,
                 local_links = self.local_links + o.local_links,
+                extras = self.extras + o.extras,
                 )
         return retval
 
     def createMaster(self, master_dir, buildbot):
         null = open(os.devnull, "w")
         subprocess.check_call([buildbot, 'create-master', master_dir], stdout=null)
         if not os.path.exists(master_dir):
             os.makedirs(master_dir)
@@ -43,22 +49,87 @@ class MasterConfig:
             os.symlink(src, dst)
 
         for src, dst in self.renames:
             dst = os.path.join(master_dir, dst)
             if os.path.lexists(dst):
                 os.unlink(dst)
             shutil.copy(os.path.join(self.config_dir, src), dst)
 
+        for extra_filename, extra_data in self.extras:
+            f = open(os.path.join(master_dir, extra_filename), 'w').write(extra_data)
+
         # Remove leftover files
         for f in "Makefile.sample", "master.cfg.sample":
             dst = os.path.join(master_dir, f)
             if os.path.exists(dst):
                 os.unlink(dst)
 
+def load_masters_json(masters_json):
+    if 'http' in masters_json:
+        masters = json.load(urllib.urlopen(masters_json))
+    else:
+        masters = json.load(open(masters_json))
+
+    retval = []
+    for m in masters:
+        # Unsupported...for now!
+        if m['role'] in ('scheduler',):
+            continue
+
+        if m['environment'] == 'production':
+            environment_config = 'production_config.py'
+        elif m['environment'] == 'staging':
+            environment_config = 'staging_config.py'
+        elif m['environment'] == 'preproduction':
+            environment_config = 'preproduction_config.py'
+        c = MasterConfig(name=m['name'],
+                globs=[
+                    'config.py',
+                    environment_config,
+                    'master_common.py',
+                    'project_branches.py',
+                    ],
+                renames=[
+                    ('BuildSlaves.py.template', 'BuildSlaves.py'),
+                    ('passwords.py.template', 'passwords.py'),
+                    ],
+                local_links=[
+                    (environment_config, 'localconfig.py'),
+                    ],
+                extras=[
+                    ('master_config.json', json.dumps(m, indent=2)),
+                    ]
+                )
+
+        if m['role'] == 'build':
+            c.config_dir = 'mozilla'
+            c.globs.append('l10n-changesets*')
+            c.globs.append('release_templates')
+            c.globs.append('release-firefox*.py')
+            c.globs.append('builder_master.cfg')
+            c.globs.append('build_localconfig.py')
+            c.local_links.append(('builder_master.cfg', 'master.cfg'))
+            c.local_links.append(('build_localconfig.py', 'master_localconfig.py'))
+        elif m['role'] == 'try':
+            c.config_dir = 'mozilla'
+            c.local_links.append(('builder_master.cfg', 'master.cfg'))
+            c.local_links.append(('try_localconfig.py', 'master_localconfig.py'))
+            c.globs.append('builder_master.cfg')
+            c.globs.append('try_localconfig.py')
+        elif m['role'] == 'tests':
+            c.config_dir = 'mozilla-tests'
+            c.local_links.append(('tests_master.cfg', 'master.cfg'))
+            c.local_links.append(('tests_localconfig.py', 'master_localconfig.py'))
+            c.globs.append('tests_localconfig.py')
+            c.globs.append('tests_master.cfg')
+
+        retval.append(c)
+    return retval
+
 mozilla2_staging = MasterConfig(
         config_dir='mozilla2-staging',
         globs=['*.py', '*.cfg', '*.ini', 'l10n-changesets*'],
         renames=[
             ('BuildSlaves.py.template', 'BuildSlaves.py'),
             ],
         local_links=[],
         )
@@ -503,25 +574,26 @@ mozilla_preproduction_release_master = m
         "preprod-release-master",
         local_links = [
             ('preproduction_release_master_localconfig.py', 'master_localconfig.py'),
             ('preproduction_config.py', 'localconfig.py'),
             ('builder_master.cfg', 'master.cfg'),
             ]
         )
 
-masters = [
+# Buildbot 0.7 masters
+masters_07 = [
         mozilla2_staging1, mozilla2_staging2,
         mozilla2_1, mozilla2_2, mozilla2_3,
         debsign_production, debsign_staging,
         mobile_production, mobile_staging,
         ]
 
-# Buildbot 0.8.0 masters
-masters_080 = [
+# Buildbot 0.8 masters
+masters_08 = [
         # Build Masters
         mozilla_staging_scheduler_master_sm01,
         mozilla_staging_builder_master_sm01,
         mozilla_staging_univeral_master_sm02,
         mozilla_staging_univeral_master_sm03,
         mozilla_production_scheduler_master,
         mozilla_production_builder_master_pm01,
         mozilla_production_builder_master_pm03,
@@ -558,27 +630,30 @@ masters_080 = [
         mozilla_preproduction_builder_master,
         mozilla_preproduction_release_master,
     ]
 
 if __name__ == "__main__":
     from optparse import OptionParser
 
     parser = OptionParser(__doc__)
-    parser.set_defaults(action=None)
+    parser.set_defaults(action=None, masters_json=None)
     parser.add_option("-l", "--list", action="store_const", dest="action", const="list")
-    parser.add_option("-8", action="store_true", dest="buildbot080", default=False)
+    parser.add_option("-7", action="store_true", dest="buildbot07", default=False)
     parser.add_option("-b", "--buildbot", dest="buildbot", default="buildbot")
+    parser.add_option("-j", "--masters-json", dest="masters_json")
 
     options, args = parser.parse_args()
 
-    if options.buildbot080:
-        master_list = masters_080
+    if options.masters_json:
+        master_list = load_masters_json(options.masters_json)
+    elif options.buildbot07:
+        master_list = masters_07
     else:
-        master_list = masters
+        master_list = masters_08
 
     # Make sure we don't have duplicate names
     master_map = dict((m.name, m) for m in master_list)
     assert len(master_map.values()) == len(master_list), "Duplicate master names"
 
     if options.action == "list":
         for m in master_list:
             if m.name != 'preprod-release-master':
--- a/test-masters.sh
+++ b/test-masters.sh
@@ -20,18 +20,22 @@
 set -e
 if [ -n "$TEMP" ]; then
     master_dir=$TEMP/master_dir
 else
     master_dir=master_dir
 fi
 # $extra_args determines if you will iterate through the 0.8.x
 # based masters or the 0.7.x ones
-if [ "$1" = "-8" ]; then
-    extra_args=-8
+if [ "$1" = "-7" ]; then
+    extra_args=-7
+
+# Any other arg will be treated as a json file to load
+elif [ -n "$1" ]; then
+    extra_args="-j $1"
 fi
 
 exit_code=0
 
 # It will iterate through list of masters and checkconfig for each one of them
 for master_name in $(python setup-master.py $extra_args --list); do
     rm -rf $master_dir
     mkdir $master_dir