testing: vastly improve support for running hg.mozilla.org clusters
authorGregory Szorc <gps@mozilla.com>
Mon, 09 Mar 2015 22:20:17 -0700
changeset 360510 b6aded8e5fada0248b593d2e6de65ca3daf935cd
parent 360509 d65b53c26da2d2663a97608f720b65dfbbf5fbe8
child 360511 f4acefded15df7555086cc703f3efdc0b9db48ac
push id16998
push userrwood@mozilla.com
push dateMon, 02 May 2016 19:42:03 +0000
testing: vastly improve support for running hg.mozilla.org clusters Support for running an hg.mozilla.org has been vastly improved. An "hgmo" mach driver for controlling hg.mozilla.org instances has been created. There is a new API behind it for controlling instances. The hgmaster Docker environment has been greatly improved. It now integrates with an OpenLDAP server. OpenSSH LPK is installed and SSH logins are processed through LDAP lookups. It "just works." SSH requests into the container result in pash successfully being invoked. Support for running hgweb containers has been added. It isn't super well-tested right now. But the master container can SSH into hgweb containers as the hg user and get the desired mirror-pull behavior. Although, things have not been tested to verify mirror-pull and other various SSH connectivity works as expected. Things are probably significantly broken.
hgmo
testing/docker/builder-hgmaster/Dockerfile
testing/docker/builder-hgmaster/entrypoint.py
testing/docker/builder-hgmaster/nslcd.conf
testing/docker/builder-hgmaster/set-mirrors.py
testing/docker/builder-hgmaster/sshd_config
testing/docker/builder-hgweb/Dockerfile
testing/docker/builder-hgweb/entrypoint.py
testing/docker/builder-hgweb/hgrc
testing/docker/builder-hgweb/hgweb.config
testing/docker/builder-hgweb/hgweb.wsgi
testing/docker/builder-hgweb/httpd.conf
testing/docker/builder-hgweb/lockfile
testing/docker/builder-hgweb/mirror-pull
testing/docker/builder-hgweb/mod_wsgi.conf
testing/docker/builder-hgweb/mozilla.repo
testing/docker/builder-hgweb/repo-group
testing/docker/builder-hgweb/run.sh
testing/docker/builder-hgweb/set-mirror-key.py
testing/docker/builder-hgweb/ssh-config
testing/docker/builder-hgweb/vhost.conf
testing/vcttesting/docker.py
testing/vcttesting/hgmo.py
testing/vcttesting/hgmo_mach_commands.py
new file mode 100755
--- /dev/null
+++ b/hgmo
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+def main(args):
+    if 'VIRTUAL_ENV' not in os.environ:
+        activate = os.path.join(HERE, 'venv', 'bin', 'activate_this.py')
+        execfile(activate, dict(__file__=activate))
+        sys.executable = os.path.join(HERE, 'venv', 'bin', 'python')
+
+    from mach.main import Mach
+
+    m = Mach(os.getcwd())
+
+    m.define_category('hgmo', 'hg.mozilla.org',
+        'Control and Interact with an hg.mozilla.org cluster', 50)
+    import vcttesting.hgmo_mach_commands
+
+    return m.run(args)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/testing/docker/builder-hgmaster/Dockerfile
+++ b/testing/docker/builder-hgmaster/Dockerfile
@@ -7,48 +7,48 @@
 # server. It installs Mercurial, adds some users and groups, and scripts
 # that mirror production (or what production will be). Should accompany
 # the hgweb-slave script.
 
 FROM centos:centos6
 MAINTAINER Ben Kero <bkero@mozilla.com>
 
 # Install requisite packages
-RUN yum install -y update; yum clean all
-RUN yum install -y openssh-server git sudo; yum clean all
-RUN yum localinstall -y http://people.mozilla.org/~bkero/mercurial-3.2.1+2-3246801ff313.x86_64.rpm
+RUN yum update -y
+
+RUN curl https://mrepo.mozilla.org/mrepo/6Server-x86_64/RPMS.mozilla/openssh-lpk-5.4p1-1.x86_64.rpm > /openssh-lpk.rpm
+RUN curl https://mrepo.mozilla.org/mrepo/6Server-x86_64/RPMS.mozilla/openssh-lpk-clients-5.4p1-1.x86_64.rpm > /openssh-lpk-clients.rpm
+RUN curl https://mrepo.mozilla.org/mrepo/6Server-x86_64/RPMS.mozilla/openssh-lpk-server-5.4p1-1.x86_64.rpm > /openssh-lpk-server.rpm
+
+RUN yum install -y authconfig nss-pam-ldapd openldap-clients pam_ldap python-ldap sudo syslog
+RUN yum install -y /openssh-lpk.rpm /openssh-lpk-clients.rpm /openssh-lpk-server.rpm
 
-# TODO: Use the virtualenv instead of global package
-#RUN yum -y install epel-release python-pip python-virtualenv gcc
-#RUN mkdir /hg; cd /hg; virtualenv venv
-#RUN cd /hg; source venv/bin/activate; pip install mercurial
+RUN yum localinstall -y https://mrepo.mozilla.org/mrepo/6Server-x86_64/RPMS.mozilla/mercurial-3.3.2-0.x86_64.rpm
+
+RUN yum clean all
 
-# Add an example user for testing (guest/guest, member of scm_level_1)
-RUN groupadd scm_level_1; groupadd scm_level_2; groupadd scm_level_3
-RUN useradd guest; echo guest:guest|chpasswd; gpasswd -a guest scm_level_1
-RUN mkdir -p /home/guest; chown guest:guest /home/guest
+# These should stay in sync with mozilla.ldif from the ldap server.
+RUN groupadd -g 600 scm_level_1
+RUN groupadd -g 601 scm_level_2
+RUN groupadd -g 602 scm_level_3
 
 # Add the hg user
 RUN useradd hg
 
 # Set root password to something for debugging purposes
 RUN echo root:mozillafirefox|chpasswd
 
 # Set up Mercurial, adding example content
 RUN mkdir -p /etc/mercurial
-RUN mkdir -p /repo_local/mozilla/mozilla /repo_local/mozilla/extensions /repo_local/mozilla/scripts /repo
-RUN ln -sf /repo_local/mozilla /repo/hg
-
-# Clone the xforms repo as a small example repository
-RUN hg clone -U https://hg.mozilla.org/xforms /repo/hg/mozilla/xforms; chmod -R 2775 /repo/hg/mozilla/xforms; chown -R hg:scm_level_1 /repo/hg/mozilla/xforms
+RUN mkdir -p /repo/hg/mozilla /repo/hg/extensions /repo/hg/scripts
 
 # Set up users directories, make permissions right
-RUN mkdir -p /repo_local/mozilla/mozilla/users
-RUN chmod 2775 /repo_local/mozilla/mozilla/users
-RUN chown -R hg:scm_level_1 /repo_local/mozilla/mozilla/users
+RUN mkdir -p /repo/hg/mozilla/users
+RUN chmod 2775 /repo/hg/mozilla/users
+RUN chown -R hg:scm_level_1 /repo/hg/mozilla/users
 
 # Install Mercurial config files
 ADD extra/vct/testing/docker/builder-hgmaster/hgrc /etc/mercurial/hgrc
 
 # TODO: Write up the client and re-enable the mirror push script
 RUN sed -i 's/changegroup.mirrorpush/#changegroup.mirrorpush/' /etc/mercurial/hgrc
 
 # Copy scripts into place
@@ -57,26 +57,29 @@ ADD extra/vct/hgext/serverlog /repo/hg/e
 ADD extra/vct/hghooks /repo/hg/libraries/
 ADD extra/vct/scripts/record-pushes.sh /repo/hg/scripts/
 ADD extra/vct/scripts/push-repo.sh /repo/hg/scripts/
 ADD extra/vct/scripts/repo-push.sh /usr/local/bin/
 
 # TODO: Fix this logging or remove record-pushes script as it's information duplicates serverlog
 RUN touch /var/log/hg-push.log; chmod 666 /var/log/hg-push.log
 
+RUN touch /var/log/pash.log && chmod 666 /var/log/pash.log
+
 # Put the files in the right place
 ADD extra/vct/scripts/pash/* /usr/local/bin/
 
-# TODO: Set up an LDAP container, run tests against it
-# Comment out the ldap nastiness
-#RUN sed -i 's/, ldap_helper//; s/ .*ldap_helper.*/           pass/' /usr/local/bin/pash.py
-#RUN sed -i 's/^from ldap_helper.*//; s/    account_status =.*/    return 1/' /usr/local/bin/hg_helper.py
+ADD sshd_config /etc/ssh/sshd_config
+ADD nslcd.conf /etc/nslcd.conf
 
-# Configure sshd to use pash.py
-RUN sed -i 's/#\tForceCommand.*/ForceCommand \/usr\/local\/bin\/pash.py/' /etc/ssh/sshd_config
-RUN sed -i 's/#PermitRootLogin/PermitRootLogin/' /etc/ssh/sshd_config
+RUN ssh-keygen -b 2048 -f /etc/mercurial/mirror -t rsa
+RUN mkdir /home/hg/.ssh
+RUN cp /etc/mercurial/mirror.pub /home/hg/.ssh/authorized_keys
+RUN chown -R hg:hg /home/hg/.ssh
+RUN chmod 640 /home/hg/.ssh/authorized_keys
 
-# Generate the SSH host keys
-RUN service sshd start
-
+ADD set-mirrors.py /set-mirrors.py
+ADD entrypoint.py /entrypoint.py
 
 EXPOSE 22
+
+ENTRYPOINT ["/entrypoint.py"]
 CMD ["/usr/sbin/sshd", "-D"]
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgmaster/entrypoint.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import sys
+
+# Disable output buffering.
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+sys.stderr = sys.stdout
+
+if 'LDAP_PORT_389_TCP_ADDR' not in os.environ:
+    print('error: container invoked improperly. please link to an ldap container')
+    sys.exit(1)
+
+ldap_hostname = os.environ['LDAP_PORT_389_TCP_ADDR']
+ldap_port = os.environ['LDAP_PORT_389_TCP_PORT']
+ldap_uri = 'ldap://%s:%s/' % (ldap_hostname, ldap_port)
+
+# Generate host SSH keys.
+if not os.path.exists('/etc/ssh/ssh_host_dsa_key'):
+    subprocess.check_call(['/usr/bin/ssh-keygen', '-t', 'dsa',
+                           '-f', '/etc/ssh/ssh_host_dsa_key'])
+
+if not os.path.exists('/etc/ssh/ssh_host_rsa_key'):
+    subprocess.check_call(['/usr/bin/ssh-keygen', '-t', 'rsa', '-b', '2048',
+                           '-f', '/etc/ssh/ssh_host_rsa_key'])
+
+# System wide LDAP configuration
+with open('/etc/openldap/ldap.conf', 'wb') as fh:
+    fh.write('\n'.join([
+        'BASE dc=mozilla',
+        'URI %s' % ldap_uri,
+    ]))
+
+# Configure PAM LDAP
+with open('/etc/pam_ldap.conf', 'wb') as fh:
+    fh.write('\n'.join([
+        'base dc=mozilla',
+        'binddn cn=admin,dc=mozilla',
+        'bindpw password',
+        'uri %s' % ldap_uri,
+        'ssl off',
+        'pam_password md5',
+    ]))
+
+# Update the LDAP server in OpenSSH config.
+sshd_config = open('/etc/ssh/sshd_config', 'rb').readlines()
+with open('/etc/ssh/sshd_config', 'wb') as fh:
+    for line in sshd_config:
+        if line.startswith('LpkServers'):
+            line = 'LpkServers %s\n' % ldap_uri
+
+        fh.write(line)
+
+nslcd_config = open('/etc/nslcd.conf', 'rb').readlines()
+with open('/etc/nslcd.conf', 'wb') as fh:
+    for line in nslcd_config:
+        if line.startswith('uri '):
+            line = 'uri %s\n' % ldap_uri
+
+        fh.write(line)
+
+REPLACEMENTS = {
+    "<%= scope.function_hiera(['secrets_openldap_moco_bindhg_username']) %>": 'cn=admin,dc=mozilla',
+    "<%= scope.function_hiera(['secrets_openldap_moco_bindhg_password']) %>": 'password',
+    "<%= scope.lookupvar('::ldapvip') %>": '%s:%s/' % (ldap_hostname, ldap_port),
+}
+
+ldap_helper = open('/usr/local/bin/ldap_helper.py', 'rb').readlines()
+with open('/usr/local/bin/ldap_helper.py', 'wb') as fh:
+    for line in ldap_helper:
+        for s, r in REPLACEMENTS.items():
+            line = line.replace(s, r)
+
+        fh.write(line)
+
+hg_helper = open('/usr/local/bin/hg_helper.py', 'rb').readlines()
+with open('/usr/local/bin/hg_helper.py', 'wb') as fh:
+    for line in hg_helper:
+        line = line.replace('ldap://ldap.db.scl3.mozilla.com', ldap_uri)
+        fh.write(line)
+
+pash = open('/usr/local/bin/pash.py', 'rb').readlines()
+with open('/usr/local/bin/pash.py', 'wb') as fh:
+    for line in pash:
+        line = line.replace('ldap://ldap.db.scl3.mozilla.com', ldap_uri)
+        line = line.replace('ldap://ldapsync1.db.scl3.mozilla.com', ldap_uri)
+        fh.write(line)
+
+subprocess.check_call([
+    '/usr/sbin/authconfig',
+    '--enablemkhomedir',
+    '--enableldap',
+    '--enableldapauth',
+    '--ldapserver=%s' % ldap_uri,
+    '--ldapbasedn=dc=mozilla',
+    '--updateall'])
+
+subprocess.check_call(['/sbin/service', 'rsyslog', 'start'])
+
+os.execl(sys.argv[1], *sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgmaster/nslcd.conf
@@ -0,0 +1,16 @@
+# This will get rewritten during entrypoint.
+uri ldap://localhost
+
+base dc=mozilla
+binddn cn=admin,dc=mozilla
+bindpw password
+
+scope sub
+bind_timelimit 30
+
+map passwd uid mail
+map passwd homeDirectory fakeHome
+
+uid nslcd
+gid ldap
+ssl no
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgmaster/set-mirrors.py
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+with open('/etc/mercurial/mirrors', 'wb') as fh:
+    for mirror in sys.argv[1:]:
+        fh.write(mirror)
+        fh.write('\n')
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgmaster/sshd_config
@@ -0,0 +1,19 @@
+LogLevel VERBOSE
+PermitRootLogin without-password
+PasswordAuthentication no
+Protocol 2
+
+AcceptEnv LANG LC_ALL LC_MESSAGES
+ChallengeResponseAuthentication no
+
+UseLPK yes
+# This entry gets replaced by the entrypoint script.
+LpkServers ldap://localhost/
+LpkForceTLS no
+LpkUIDAttribute mail
+LpkBindDN cn=admin,dc=mozilla
+LpkBindPw password
+LpkUserDN dc=mozilla
+
+Subsystem sftp /usr/libexec/openssh/sftp-server
+ForceCommand /usr/local/bin/pash.py
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/Dockerfile
@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM centos:centos6
+
+RUN useradd -u 500 hg
+
+ADD mozilla.repo /etc/yum.repos.d/mozilla.repo
+RUN yum update -y
+
+RUN mkdir -p /repo_local/mozilla/mozilla /repo
+RUN ln -sf /repo_local/mozilla /repo/hg
+RUN chown -R hg:hg /repo_local/mozilla/mozilla
+
+RUN yum install -y httpd lockfile-progs mod_wsgi openssh-server python-pygments python-simplejson python-argparse syslog
+
+RUN yum localinstall -y https://mrepo.mozilla.org/mrepo/6Server-x86_64/RPMS.mozilla/mercurial-3.3.2-0.x86_64.rpm
+
+RUN yum clean all
+
+RUN hg clone http://hg.mozilla.org/hgcustom/library_overrides /repo_local/mozilla/library_overrides
+
+ADD httpd.conf /etc/httpd/conf/httpd.conf
+ADD vhost.conf /etc/httpd/conf.d/hg-vhost.conf
+ADD mod_wsgi.conf /etc/httpd/conf.d/wsgi.conf
+RUN mkdir /var/log/httpd/hg
+
+ADD hgrc /etc/mercurial/hgrc
+ADD mirror-pull /usr/local/bin/mirror-pull
+ADD repo-group /usr/local/bin/repo-group
+ADD lockfile /usr/local/bin/lockfile
+ADD ssh-config /home/hg/.ssh/config
+
+ADD extra/vct/hgtemplates /repo_local/mozilla/hg_templates
+ADD extra/vct/hghooks /repo_local/mozilla/hghooks
+
+ADD extra/vct/hgext/pushlog-legacy/buglink.py /repo_local/mozilla/extensions
+ADD extra/vct/hgext/pushlog-legacy/feedparser.py /repo_local/mozilla/extensions
+ADD extra/vct/hgext/pushlog-legacy/hgwebjson.py /repo_local/mozilla/extensions
+ADD extra/vct/hgext/pushlog-legacy/pushlog-feed.py /repo_local/mozilla/extensions
+RUN ln -sf /repo_local/mozilla/hghooks/mozhghooks /repo_local/mozilla/libraries
+
+ADD hgweb.wsgi /repo_local/mozilla/webroot_wsgi/hgweb.wsgi
+ADD hgweb.config /repo_local/mozilla/webroot_wsgi/hgweb.config
+ADD set-mirror-key.py /set-mirror-key.py
+ADD entrypoint.py /entrypoint.py
+ADD run.sh /run.sh
+
+EXPOSE 22
+EXPOSE 80
+
+ENTRYPOINT ["/entrypoint.py"]
+CMD ["/run.sh"]
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgweb/entrypoint.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import sys
+
+# Disable output buffering.
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+sys.stderr = sys.stdout
+
+if 'MASTER_PORT_22_TCP_ADDR' not in os.environ:
+    print('error: container invoked improperly. please link to a master container')
+    sys.exit(1)
+
+if not os.path.exists('/etc/ssh/ssh_host_dsa_key'):
+    subprocess.check_call(['/usr/bin/ssh-keygen', '-t', 'dsa',
+                           '-f', '/etc/ssh/ssh_host_dsa_key'])
+
+if not os.path.exists('/etc/ssh/ssh_host_rsa_key'):
+    subprocess.check_call(['/usr/bin/ssh-keygen', '-t', 'rsa', '-b', '2048',
+                           '-f', '/etc/ssh/ssh_host_rsa_key'])
+
+REPLACEMENTS = {
+    '<%= @mirror_source %>': os.environ['MASTER_PORT_22_TCP_ADDR'],
+    '<%= @repo_serve_path %>': '/repo_local/mozilla/mozilla',
+    '%<= @repo_serve_path %>': '/repo_local/mozilla/mozilla',
+    '<%= @python_lib_override_path %>': '/repo_local/mozilla/library_overrides',
+    '<%= @python_lib_path %>': '/repo_local/mozilla/libraries',
+    '<%= @mirror_priv_key_path %>': '/etc/mercurial/mirror',
+    '<%= @mirror_user_name %>': 'hg',
+}
+
+mirror_pull = open('/usr/local/bin/mirror-pull', 'rb').readlines()
+with open('/usr/local/bin/mirror-pull', 'wb') as fh:
+    for line in mirror_pull:
+        for k, v in REPLACEMENTS.items():
+            line = line.replace(k, v)
+
+        fh.write(line)
+
+os.execl(sys.argv[1], *sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/hgrc
@@ -0,0 +1,42 @@
+####################################
+## This file is managed by puppet ##
+####################################
+
+[trusted]
+users = hg
+
+[email]
+method = /usr/sbin/sendmail
+from = hg@hg
+
+[web]
+logourl = https://developer.mozilla.org/en/docs/Mercurial
+style = gitweb_mozilla
+allow_archive = bz2 gz zip
+templates = /repo_local/mozilla/hg_templates
+pygments_style = emacs
+encoding = UTF-8
+baseurl = http://hg
+maxchanges = 20
+guessmime = True
+
+[diff]
+showfunc = True
+git = 1
+unified = 8
+
+[extensions]
+hgwebjson = /repo_local/mozilla/extensions/hgwebjson.py
+pushlog-feed = /repo_local/mozilla/extensions/pushlog-feed.py
+pushlog = /repo_local/mozilla/extensions/pushlog
+buglink = /repo_local/mozilla/extensions/buglink.py
+serverlog = /repo_local/mozilla/extensions/serverlog
+#highlight =
+#interhg =
+
+[interhg]
+a_httplinks = s|(https?://\S+)|<a href="\1">\1</a>|i
+bugzilla = s|((?:bug[\s#]*|b=#?|#)(\d{4,}))|<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=\2" title="Bug \2">\1</a>|i
+
+#[serverlog]
+#reporoot = /repo_local/mozilla/
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/hgweb.config
@@ -0,0 +1,4 @@
+[web]
+style = gitweb_mozilla
+
+[paths]
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/hgweb.wsgi
@@ -0,0 +1,6 @@
+config = "/repo_local/mozilla/webroot_wsgi/hgweb.config"
+from mercurial import demandimport; demandimport.enable()
+from mercurial.hgweb import hgweb
+import os
+os.environ["HGENCODING"] = "UTF-8"
+application = hgweb(config)
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/httpd.conf
@@ -0,0 +1,1012 @@
+#
+# This is the main Apache server configuration file.  It contains the
+# configuration directives that give the server its instructions.
+# See <URL:http://httpd.apache.org/docs/2.2/> for detailed information.
+# In particular, see
+# <URL:http://httpd.apache.org/docs/2.2/mod/directives.html>
+# for a discussion of each configuration directive.
+#
+#
+# Do NOT simply read the instructions in here without understanding
+# what they do.  They're here only as hints or reminders.  If you are unsure
+# consult the online docs. You have been warned.  
+#
+# The configuration directives are grouped into three basic sections:
+#  1. Directives that control the operation of the Apache server process as a
+#     whole (the 'global environment').
+#  2. Directives that define the parameters of the 'main' or 'default' server,
+#     which responds to requests that aren't handled by a virtual host.
+#     These directives also provide default values for the settings
+#     of all virtual hosts.
+#  3. Settings for virtual hosts, which allow Web requests to be sent to
+#     different IP addresses or hostnames and have them handled by the
+#     same Apache server process.
+#
+# Configuration and logfile names: If the filenames you specify for many
+# of the server's control files begin with "/" (or "drive:/" for Win32), the
+# server will use that explicit path.  If the filenames do *not* begin
+# with "/", the value of ServerRoot is prepended -- so "logs/foo.log"
+# with ServerRoot set to "/etc/httpd" will be interpreted by the
+# server as "/etc/httpd/logs/foo.log".
+#
+
+### Section 1: Global Environment
+#
+# The directives in this section affect the overall operation of Apache,
+# such as the number of concurrent requests it can handle or where it
+# can find its configuration files.
+#
+
+#
+# Don't give away too much information about all the subcomponents
+# we are running.  Comment out this line if you don't mind remote sites
+# finding out what major optional modules you are running
+#ServerTokens OS
+ServerTokens Prod
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# NOTE!  If you intend to place this on an NFS (or otherwise network)
+# mounted filesystem then please read the LockFile documentation
+# (available at <URL:http://httpd.apache.org/docs/2.2/mod/mpm_common.html#lockfile>);
+# you will save yourself a lot of trouble.
+#
+# Do NOT add a slash at the end of the directory path.
+#
+ServerRoot "/etc/httpd"
+
+#
+# PidFile: The file in which the server should record its process
+# identification number when it starts.  Note the PIDFILE variable in
+# /etc/sysconfig/httpd must be set appropriately if this location is
+# changed.
+#
+PidFile run/httpd.pid
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 60
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive Off
+
+#
+# MaxKeepAliveRequests: The maximum number of requests to allow
+# during a persistent connection. Set to 0 to allow an unlimited amount.
+# We recommend you leave this number high, for maximum performance.
+#
+MaxKeepAliveRequests 100
+
+#
+# KeepAliveTimeout: Number of seconds to wait for the next request from the
+# same client on the same connection.
+#
+KeepAliveTimeout 15
+
+##
+## Server-Pool Size Regulation (MPM specific)
+## 
+
+# prefork MPM
+# StartServers: number of server processes to start
+# MinSpareServers: minimum number of server processes which are kept spare
+# MaxSpareServers: maximum number of server processes which are kept spare
+# ServerLimit: maximum value for MaxClients for the lifetime of the server
+# MaxClients: maximum number of server processes allowed to start
+# MaxRequestsPerChild: maximum number of requests a server process serves
+<IfModule prefork.c>
+StartServers       8
+MinSpareServers    5
+MaxSpareServers   20
+ServerLimit      256
+MaxClients       256
+MaxRequestsPerChild  4000
+</IfModule>
+
+# worker MPM
+# StartServers: initial number of server processes to start
+# MaxClients: maximum number of simultaneous client connections
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestsPerChild: maximum number of requests a server process serves
+<IfModule worker.c>
+StartServers         4
+MaxClients         300
+MinSpareThreads     25
+MaxSpareThreads     75 
+ThreadsPerChild     25
+MaxRequestsPerChild  0
+</IfModule>
+
+#
+# Listen: Allows you to bind Apache to specific IP addresses and/or
+# ports, in addition to the default. See also the <VirtualHost>
+# directive.
+#
+# Change this to Listen on specific IP addresses as shown below to 
+# prevent Apache from glomming onto all bound IP addresses (0.0.0.0)
+#
+#Listen 12.34.56.78:80
+Listen 80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule auth_digest_module modules/mod_auth_digest.so
+LoadModule authn_file_module modules/mod_authn_file.so
+LoadModule authn_alias_module modules/mod_authn_alias.so
+LoadModule authn_anon_module modules/mod_authn_anon.so
+LoadModule authn_dbm_module modules/mod_authn_dbm.so
+LoadModule authn_default_module modules/mod_authn_default.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule authz_user_module modules/mod_authz_user.so
+LoadModule authz_owner_module modules/mod_authz_owner.so
+LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+LoadModule authz_dbm_module modules/mod_authz_dbm.so
+LoadModule authz_default_module modules/mod_authz_default.so
+LoadModule ldap_module modules/mod_ldap.so
+LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+LoadModule include_module modules/mod_include.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule logio_module modules/mod_logio.so
+LoadModule env_module modules/mod_env.so
+LoadModule ext_filter_module modules/mod_ext_filter.so
+LoadModule mime_magic_module modules/mod_mime_magic.so
+LoadModule expires_module modules/mod_expires.so
+LoadModule deflate_module modules/mod_deflate.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule usertrack_module modules/mod_usertrack.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule dav_module modules/mod_dav.so
+LoadModule status_module modules/mod_status.so
+LoadModule autoindex_module modules/mod_autoindex.so
+LoadModule info_module modules/mod_info.so
+LoadModule dav_fs_module modules/mod_dav_fs.so
+LoadModule vhost_alias_module modules/mod_vhost_alias.so
+LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule dir_module modules/mod_dir.so
+LoadModule actions_module modules/mod_actions.so
+LoadModule speling_module modules/mod_speling.so
+LoadModule userdir_module modules/mod_userdir.so
+LoadModule alias_module modules/mod_alias.so
+LoadModule substitute_module modules/mod_substitute.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+LoadModule proxy_ftp_module modules/mod_proxy_ftp.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule proxy_ajp_module modules/mod_proxy_ajp.so
+LoadModule proxy_connect_module modules/mod_proxy_connect.so
+LoadModule cache_module modules/mod_cache.so
+LoadModule suexec_module modules/mod_suexec.so
+LoadModule disk_cache_module modules/mod_disk_cache.so
+LoadModule cgi_module modules/mod_cgi.so
+LoadModule version_module modules/mod_version.so
+
+#
+# The following modules are not loaded by default:
+#
+#LoadModule asis_module modules/mod_asis.so
+#LoadModule authn_dbd_module modules/mod_authn_dbd.so
+#LoadModule cern_meta_module modules/mod_cern_meta.so
+#LoadModule cgid_module modules/mod_cgid.so
+#LoadModule dbd_module modules/mod_dbd.so
+#LoadModule dumpio_module modules/mod_dumpio.so
+#LoadModule filter_module modules/mod_filter.so
+#LoadModule ident_module modules/mod_ident.so
+#LoadModule log_forensic_module modules/mod_log_forensic.so
+#LoadModule unique_id_module modules/mod_unique_id.so
+#
+
+#
+# Load config files from the config directory "/etc/httpd/conf.d".
+#
+Include conf.d/*.conf
+
+#
+# ExtendedStatus controls whether Apache will generate "full" status
+# information (ExtendedStatus On) or just basic information (ExtendedStatus
+# Off) when the "server-status" handler is called. The default is Off.
+#
+#ExtendedStatus On
+
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.  
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+#  . On SCO (ODT 3) use "User nouser" and "Group nogroup".
+#  . On HPUX you may not be able to use shared memory as nobody, and the
+#    suggested workaround is to create a user www and use that user.
+#  NOTE that some kernels refuse to setgid(Group) or semctl(IPC_SET)
+#  when the value of (unsigned)Group is above 60000; 
+#  don't use Group #-1 on these systems!
+#
+User apache
+Group apache
+
+### Section 2: 'Main' server configuration
+#
+# The directives in this section set up the values used by the 'main'
+# server, which responds to any requests that aren't handled by a
+# <VirtualHost> definition.  These values also provide defaults for
+# any <VirtualHost> containers you may define later in the file.
+#
+# All of these directives may appear inside <VirtualHost> containers,
+# in which case these default settings will be overridden for the
+# virtual host being defined.
+#
+
+#
+# ServerAdmin: Your address, where problems with the server should be
+# e-mailed.  This address appears on some server-generated pages, such
+# as error documents.  e.g. admin@your-domain.com
+#
+ServerAdmin root@localhost
+
+#
+# ServerName gives the name and port that the server uses to identify itself.
+# This can often be determined automatically, but we recommend you specify
+# it explicitly to prevent problems during startup.
+#
+# If this is not set to valid DNS name for your host, server-generated
+# redirections will not work.  See also the UseCanonicalName directive.
+#
+# If your host doesn't have a registered DNS name, enter its IP address here.
+# You will have to access it by its address anyway, and this will make 
+# redirections work in a sensible way.
+#
+#ServerName www.example.com:80
+
+#
+# UseCanonicalName: Determines how Apache constructs self-referencing 
+# URLs and the SERVER_NAME and SERVER_PORT variables.
+# When set "Off", Apache will use the Hostname and Port supplied
+# by the client.  When set "On", Apache will use the value of the
+# ServerName directive.
+#
+UseCanonicalName Off
+
+#
+# DocumentRoot: The directory out of which you will serve your
+# documents. By default, all requests are taken from this directory, but
+# symbolic links and aliases may be used to point to other locations.
+#
+DocumentRoot "/var/www/html"
+
+#
+# Each directory to which Apache has access can be configured with respect
+# to which services and features are allowed and/or disabled in that
+# directory (and its subdirectories). 
+#
+# First, we configure the "default" to be a very restrictive set of 
+# features.  
+#
+<Directory />
+    Options FollowSymLinks
+    AllowOverride None
+</Directory>
+
+#
+# Note that from this point forward you must specifically allow
+# particular features to be enabled - so if something's not working as
+# you might expect, make sure that you have specifically enabled it
+# below.
+#
+
+#
+# This should be changed to whatever you set DocumentRoot to.
+#
+<Directory "/var/www/html">
+
+#
+# Possible values for the Options directive are "None", "All",
+# or any combination of:
+#   Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
+#
+# Note that "MultiViews" must be named *explicitly* --- "Options All"
+# doesn't give it to you.
+#
+# The Options directive is both complicated and important.  Please see
+# http://httpd.apache.org/docs/2.2/mod/core.html#options
+# for more information.
+#
+    Options Indexes FollowSymLinks
+
+#
+# AllowOverride controls what directives may be placed in .htaccess files.
+# It can be "All", "None", or any combination of the keywords:
+#   Options FileInfo AuthConfig Limit
+#
+    AllowOverride None
+
+#
+# Controls who can get stuff from this server.
+#
+    Order allow,deny
+    Allow from all
+
+</Directory>
+
+#
+# UserDir: The name of the directory that is appended onto a user's home
+# directory if a ~user request is received.
+#
+# The path to the end user account 'public_html' directory must be
+# accessible to the webserver userid.  This usually means that ~userid
+# must have permissions of 711, ~userid/public_html must have permissions
+# of 755, and documents contained therein must be world-readable.
+# Otherwise, the client will only receive a "403 Forbidden" message.
+#
+# See also: http://httpd.apache.org/docs/misc/FAQ.html#forbidden
+#
+<IfModule mod_userdir.c>
+    #
+    # UserDir is disabled by default since it can confirm the presence
+    # of a username on the system (depending on home directory
+    # permissions).
+    #
+    UserDir disabled
+
+    #
+    # To enable requests to /~user/ to serve the user's public_html
+    # directory, remove the "UserDir disabled" line above, and uncomment
+    # the following line instead:
+    # 
+    #UserDir public_html
+
+</IfModule>
+
+#
+# Control access to UserDir directories.  The following is an example
+# for a site where these directories are restricted to read-only.
+#
+#<Directory /home/*/public_html>
+#    AllowOverride FileInfo AuthConfig Limit
+#    Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec
+#    <Limit GET POST OPTIONS>
+#        Order allow,deny
+#        Allow from all
+#    </Limit>
+#    <LimitExcept GET POST OPTIONS>
+#        Order deny,allow
+#        Deny from all
+#    </LimitExcept>
+#</Directory>
+
+#
+# DirectoryIndex: sets the file that Apache will serve if a directory
+# is requested.
+#
+# The index.html.var file (a type-map) is used to deliver content-
+# negotiated documents.  The MultiViews Option can be used for the 
+# same purpose, but it is much slower.
+#
+DirectoryIndex index.html index.html.var
+
+#
+# AccessFileName: The name of the file to look for in each directory
+# for additional configuration directives.  See also the AllowOverride
+# directive.
+#
+AccessFileName .htaccess
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being 
+# viewed by Web clients. 
+#
+<Files ~ "^\.ht">
+    Order allow,deny
+    Deny from all
+    Satisfy All
+</Files>
+
+#
+# TypesConfig describes where the mime.types file (or equivalent) is
+# to be found.
+#
+TypesConfig /etc/mime.types
+
+#
+# DefaultType is the default MIME type the server will use for a document
+# if it cannot otherwise determine one, such as from filename extensions.
+# If your server contains mostly text or HTML documents, "text/plain" is
+# a good value.  If most of your content is binary, such as applications
+# or images, you may want to use "application/octet-stream" instead to
+# keep browsers from trying to display binary files as though they are
+# text.
+#
+DefaultType text/plain
+
+#
+# The mod_mime_magic module allows the server to use various hints from the
+# contents of the file itself to determine its type.  The MIMEMagicFile
+# directive tells the module where the hint definitions are located.
+#
+<IfModule mod_mime_magic.c>
+#   MIMEMagicFile /usr/share/magic.mime
+    MIMEMagicFile conf/magic
+</IfModule>
+
+#
+# HostnameLookups: Log the names of clients or just their IP addresses
+# e.g., www.apache.org (on) or 204.62.129.132 (off).
+# The default is off because it'd be overall better for the net if people
+# had to knowingly turn this feature on, since enabling it means that
+# each client request will result in AT LEAST one lookup request to the
+# nameserver.
+#
+HostnameLookups Off
+
+#
+# EnableMMAP: Control whether memory-mapping is used to deliver
+# files (assuming that the underlying OS supports it).
+# The default is on; turn this off if you serve from NFS-mounted 
+# filesystems.  On some systems, turning it off (regardless of
+# filesystem) can improve performance; for details, please see
+# http://httpd.apache.org/docs/2.2/mod/core.html#enablemmap
+#
+#EnableMMAP off
+
+#
+# EnableSendfile: Control whether the sendfile kernel support is 
+# used to deliver files (assuming that the OS supports it). 
+# The default is on; turn this off if you serve from NFS-mounted 
+# filesystems.  Please see
+# http://httpd.apache.org/docs/2.2/mod/core.html#enablesendfile
+#
+#EnableSendfile off
+
+#
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a <VirtualHost>
+# container, error messages relating to that virtual host will be
+# logged here.  If you *do* define an error logfile for a <VirtualHost>
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog logs/error_log
+
+#
+# LogLevel: Control the number of messages logged to the error_log.
+# Possible values include: debug, info, notice, warn, error, crit,
+# alert, emerg.
+#
+LogLevel warn
+
+#
+# The following directives define some format nicknames for use with
+# a CustomLog directive (see below).
+#
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %b" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+
+# "combinedio" includes actual counts of actual bytes received (%I) and sent (%O); this
+# requires the mod_logio module to be loaded.
+#LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+
+#
+# The location and format of the access logfile (Common Logfile Format).
+# If you do not define any access logfiles within a <VirtualHost>
+# container, they will be logged here.  Contrariwise, if you *do*
+# define per-<VirtualHost> access logfiles, transactions will be
+# logged therein and *not* in this file.
+#
+#CustomLog logs/access_log common
+
+#
+# If you would like to have separate agent and referer logfiles, uncomment
+# the following directives.
+#
+#CustomLog logs/referer_log referer
+#CustomLog logs/agent_log agent
+
+#
+# For a single logfile with access, agent, and referer information
+# (Combined Logfile Format), use the following directive:
+#
+CustomLog logs/access_log combined
+
+#
+# Optionally add a line containing the server version and virtual host
+# name to server-generated pages (internal error documents, FTP directory
+# listings, mod_status and mod_info output etc., but not CGI generated
+# documents or custom error documents).
+# Set to "EMail" to also include a mailto: link to the ServerAdmin.
+# Set to one of:  On | Off | EMail
+#
+#ServerSignature On
+ServerSignature Off
+
+#
+# Aliases: Add here as many aliases as you need (with no limit). The format is 
+# Alias fakename realname
+#
+# Note that if you include a trailing / on fakename then the server will
+# require it to be present in the URL.  So "/icons" isn't aliased in this
+# example, only "/icons/".  If the fakename is slash-terminated, then the 
+# realname must also be slash terminated, and if the fakename omits the 
+# trailing slash, the realname must also omit it.
+#
+# We include the /icons/ alias for FancyIndexed directory listings.  If you
+# do not use FancyIndexing, you may comment this out.
+#
+Alias /icons/ "/var/www/icons/"
+
+<Directory "/var/www/icons">
+    Options Indexes MultiViews FollowSymLinks
+    AllowOverride None
+    Order allow,deny
+    Allow from all
+</Directory>
+
+#
+# WebDAV module configuration section.
+# 
+<IfModule mod_dav_fs.c>
+    # Location of the WebDAV lock database.
+    DAVLockDB /var/lib/dav/lockdb
+</IfModule>
+
+#
+# ScriptAlias: This controls which directories contain server scripts.
+# ScriptAliases are essentially the same as Aliases, except that
+# documents in the realname directory are treated as applications and
+# run by the server when requested rather than as documents sent to the client.
+# The same rules about trailing "/" apply to ScriptAlias directives as to
+# Alias.
+#
+ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
+
+#
+# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased
+# CGI directory exists, if you have that configured.
+#
+<Directory "/var/www/cgi-bin">
+    AllowOverride None
+    Options None
+    Order allow,deny
+    Allow from all
+</Directory>
+
+#
+# Redirect allows you to tell clients about documents which used to exist in
+# your server's namespace, but do not anymore. This allows you to tell the
+# clients where to look for the relocated document.
+# Example:
+# Redirect permanent /foo http://www.example.com/bar
+
+#
+# Directives controlling the display of server-generated directory listings.
+#
+
+#
+# IndexOptions: Controls the appearance of server-generated directory
+# listings.
+#
+IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8
+
+#
+# AddIcon* directives tell the server which icon to show for different
+# files or filename extensions.  These are only displayed for
+# FancyIndexed directories.
+#
+AddIconByEncoding (CMP,/icons/compressed.gif) x-compress x-gzip
+
+AddIconByType (TXT,/icons/text.gif) text/*
+AddIconByType (IMG,/icons/image2.gif) image/*
+AddIconByType (SND,/icons/sound2.gif) audio/*
+AddIconByType (VID,/icons/movie.gif) video/*
+
+AddIcon /icons/binary.gif .bin .exe
+AddIcon /icons/binhex.gif .hqx
+AddIcon /icons/tar.gif .tar
+AddIcon /icons/world2.gif .wrl .wrl.gz .vrml .vrm .iv
+AddIcon /icons/compressed.gif .Z .z .tgz .gz .zip
+AddIcon /icons/a.gif .ps .ai .eps
+AddIcon /icons/layout.gif .html .shtml .htm .pdf
+AddIcon /icons/text.gif .txt
+AddIcon /icons/c.gif .c
+AddIcon /icons/p.gif .pl .py
+AddIcon /icons/f.gif .for
+AddIcon /icons/dvi.gif .dvi
+AddIcon /icons/uuencoded.gif .uu
+AddIcon /icons/script.gif .conf .sh .shar .csh .ksh .tcl
+AddIcon /icons/tex.gif .tex
+AddIcon /icons/bomb.gif core
+
+AddIcon /icons/back.gif ..
+AddIcon /icons/hand.right.gif README
+AddIcon /icons/folder.gif ^^DIRECTORY^^
+AddIcon /icons/blank.gif ^^BLANKICON^^
+
+#
+# DefaultIcon is which icon to show for files which do not have an icon
+# explicitly set.
+#
+DefaultIcon /icons/unknown.gif
+
+#
+# AddDescription allows you to place a short description after a file in
+# server-generated indexes.  These are only displayed for FancyIndexed
+# directories.
+# Format: AddDescription "description" filename
+#
+#AddDescription "GZIP compressed document" .gz
+#AddDescription "tar archive" .tar
+#AddDescription "GZIP compressed tar archive" .tgz
+
+#
+# ReadmeName is the name of the README file the server will look for by
+# default, and append to directory listings.
+#
+# HeaderName is the name of a file which should be prepended to
+# directory indexes. 
+ReadmeName README.html
+HeaderName HEADER.html
+
+#
+# IndexIgnore is a set of filenames which directory indexing should ignore
+# and not include in the listing.  Shell-style wildcarding is permitted.
+#
+IndexIgnore .??* *~ *# HEADER* README* RCS CVS *,v *,t
+
+#
+# DefaultLanguage and AddLanguage allows you to specify the language of 
+# a document. You can then use content negotiation to give a browser a 
+# file in a language the user can understand.
+#
+# Specify a default language. This means that all data
+# going out without a specific language tag (see below) will 
+# be marked with this one. You probably do NOT want to set
+# this unless you are sure it is correct for all cases.
+#
+# * It is generally better to not mark a page as 
+# * being a certain language than marking it with the wrong
+# * language!
+#
+# DefaultLanguage nl
+#
+# Note 1: The suffix does not have to be the same as the language
+# keyword --- those with documents in Polish (whose net-standard
+# language code is pl) may wish to use "AddLanguage pl .po" to
+# avoid the ambiguity with the common suffix for perl scripts.
+#
+# Note 2: The example entries below illustrate that in some cases 
+# the two character 'Language' abbreviation is not identical to 
+# the two character 'Country' code for its country,
+# E.g. 'Danmark/dk' versus 'Danish/da'.
+#
+# Note 3: In the case of 'ltz' we violate the RFC by using a three char
+# specifier. There is 'work in progress' to fix this and get
+# the reference data for rfc1766 cleaned up.
+#
+# Catalan (ca) - Croatian (hr) - Czech (cs) - Danish (da) - Dutch (nl)
+# English (en) - Esperanto (eo) - Estonian (et) - French (fr) - German (de)
+# Greek-Modern (el) - Hebrew (he) - Italian (it) - Japanese (ja)
+# Korean (ko) - Luxembourgeois* (ltz) - Norwegian Nynorsk (nn)
+# Norwegian (no) - Polish (pl) - Portugese (pt)
+# Brazilian Portuguese (pt-BR) - Russian (ru) - Swedish (sv)
+# Simplified Chinese (zh-CN) - Spanish (es) - Traditional Chinese (zh-TW)
+#
+AddLanguage ca .ca
+AddLanguage cs .cz .cs
+AddLanguage da .dk
+AddLanguage de .de
+AddLanguage el .el
+AddLanguage en .en
+AddLanguage eo .eo
+AddLanguage es .es
+AddLanguage et .et
+AddLanguage fr .fr
+AddLanguage he .he
+AddLanguage hr .hr
+AddLanguage it .it
+AddLanguage ja .ja
+AddLanguage ko .ko
+AddLanguage ltz .ltz
+AddLanguage nl .nl
+AddLanguage nn .nn
+AddLanguage no .no
+AddLanguage pl .po
+AddLanguage pt .pt
+AddLanguage pt-BR .pt-br
+AddLanguage ru .ru
+AddLanguage sv .sv
+AddLanguage zh-CN .zh-cn
+AddLanguage zh-TW .zh-tw
+
+#
+# LanguagePriority allows you to give precedence to some languages
+# in case of a tie during content negotiation.
+#
+# Just list the languages in decreasing order of preference. We have
+# more or less alphabetized them here. You probably want to change this.
+#
+LanguagePriority en ca cs da de el eo es et fr he hr it ja ko ltz nl nn no pl pt pt-BR ru sv zh-CN zh-TW
+
+#
+# ForceLanguagePriority allows you to serve a result page rather than
+# MULTIPLE CHOICES (Prefer) [in case of a tie] or NOT ACCEPTABLE (Fallback)
+# [in case no accepted languages matched the available variants]
+#
+ForceLanguagePriority Prefer Fallback
+
+#
+# Specify a default charset for all content served; this enables
+# interpretation of all content as UTF-8 by default.  To use the 
+# default browser choice (ISO-8859-1), or to allow the META tags
+# in HTML content to override this choice, comment out this
+# directive:
+#
+AddDefaultCharset UTF-8
+
+#
+# AddType allows you to add to or override the MIME configuration
+# file mime.types for specific file types.
+#
+#AddType application/x-tar .tgz
+
+#
+# AddEncoding allows you to have certain browsers uncompress
+# information on the fly. Note: Not all browsers support this.
+# Despite the name similarity, the following Add* directives have nothing
+# to do with the FancyIndexing customization directives above.
+#
+#AddEncoding x-compress .Z
+#AddEncoding x-gzip .gz .tgz
+
+# If the AddEncoding directives above are commented-out, then you
+# probably should define those extensions to indicate media types:
+#
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+#
+#   MIME-types for downloading Certificates and CRLs
+#
+AddType application/x-x509-ca-cert .crt
+AddType application/x-pkcs7-crl    .crl
+
+#
+# AddHandler allows you to map certain file extensions to "handlers":
+# actions unrelated to filetype. These can be either built into the server
+# or added with the Action directive (see below)
+#
+# To use CGI scripts outside of ScriptAliased directories:
+# (You will also need to add "ExecCGI" to the "Options" directive.)
+#
+#AddHandler cgi-script .cgi
+
+#
+# For files that include their own HTTP headers:
+#
+#AddHandler send-as-is asis
+
+#
+# For type maps (negotiated resources):
+# (This is enabled by default to allow the Apache "It Worked" page
+#  to be distributed in multiple languages.)
+#
+AddHandler type-map var
+
+#
+# Filters allow you to process content before it is sent to the client.
+#
+# To parse .shtml files for server-side includes (SSI):
+# (You will also need to add "Includes" to the "Options" directive.)
+#
+AddType text/html .shtml
+AddOutputFilter INCLUDES .shtml
+
+#
+# Action lets you define media types that will execute a script whenever
+# a matching file is called. This eliminates the need for repeated URL
+# pathnames for oft-used CGI file processors.
+# Format: Action media/type /cgi-script/location
+# Format: Action handler-name /cgi-script/location
+#
+
+#
+# Customizable error responses come in three flavors:
+# 1) plain text 2) local redirects 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# Putting this all together, we can internationalize error responses.
+#
+# We use Alias to redirect any /error/HTTP_<error>.html.var response to
+# our collection of by-error message multi-language collections.  We use 
+# includes to substitute the appropriate text.
+#
+# You can modify the messages' appearance without changing any of the
+# default HTTP_<error>.html.var files by adding the line:
+#
+#   Alias /error/include/ "/your/include/path/"
+#
+# which allows you to create your own set of files by starting with the
+# /var/www/error/include/ files and
+# copying them to /your/include/path/, even on a per-VirtualHost basis.
+#
+
+Alias /error/ "/var/www/error/"
+
+<IfModule mod_negotiation.c>
+<IfModule mod_include.c>
+    <Directory "/var/www/error">
+        AllowOverride None
+        Options IncludesNoExec
+        AddOutputFilter Includes html
+        AddHandler type-map var
+        Order allow,deny
+        Allow from all
+        LanguagePriority en es de fr
+        ForceLanguagePriority Prefer Fallback
+    </Directory>
+
+#    ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var
+#    ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var
+#    ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var
+#    ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var
+#    ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var
+#    ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var
+#    ErrorDocument 410 /error/HTTP_GONE.html.var
+#    ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var
+#    ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var
+#    ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var
+#    ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var
+#    ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var
+#    ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var
+#    ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var
+#    ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var
+#    ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var
+#    ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var
+
+</IfModule>
+</IfModule>
+
+#
+# The following directives modify normal HTTP response behavior to
+# handle known problems with browser implementations.
+#
+BrowserMatch "Mozilla/2" nokeepalive
+BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0
+BrowserMatch "RealPlayer 4\.0" force-response-1.0
+BrowserMatch "Java/1\.0" force-response-1.0
+BrowserMatch "JDK/1\.0" force-response-1.0
+
+#
+# The following directive disables redirects on non-GET requests for
+# a directory that does not include the trailing slash.  This fixes a 
+# problem with Microsoft WebFolders which does not appropriately handle 
+# redirects for folders with DAV methods.
+# Same deal with Apple's DAV filesystem and Gnome VFS support for DAV.
+#
+BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully
+BrowserMatch "MS FrontPage" redirect-carefully
+BrowserMatch "^WebDrive" redirect-carefully
+BrowserMatch "^WebDAVFS/1.[0123]" redirect-carefully
+BrowserMatch "^gnome-vfs/1.0" redirect-carefully
+BrowserMatch "^XML Spy" redirect-carefully
+BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
+
+#
+# Allow server status reports generated by mod_status,
+# with the URL of http://servername/server-status
+# Change the ".example.com" to match your domain to enable.
+#
+#<Location /server-status>
+#    SetHandler server-status
+#    Order deny,allow
+#    Deny from all
+#    Allow from .example.com
+#</Location>
+
+#
+# Allow remote server configuration reports, with the URL of
+#  http://servername/server-info (requires that mod_info.c be loaded).
+# Change the ".example.com" to match your domain to enable.
+#
+#<Location /server-info>
+#    SetHandler server-info
+#    Order deny,allow
+#    Deny from all
+#    Allow from .example.com
+#</Location>
+
+#
+# Proxy Server directives. Uncomment the following lines to
+# enable the proxy server:
+#
+#<IfModule mod_proxy.c>
+#ProxyRequests On
+#
+#<Proxy *>
+#    Order deny,allow
+#    Deny from all
+#    Allow from .example.com
+#</Proxy>
+
+#
+# Enable/disable the handling of HTTP/1.1 "Via:" headers.
+# ("Full" adds the server version; "Block" removes all outgoing Via: headers)
+# Set to one of: Off | On | Full | Block
+#
+#ProxyVia On
+
+#
+# To enable a cache of proxied content, uncomment the following lines.
+# See http://httpd.apache.org/docs/2.2/mod/mod_cache.html for more details.
+#
+#<IfModule mod_disk_cache.c>
+#   CacheEnable disk /
+#   CacheRoot "/var/cache/mod_proxy"
+#</IfModule>
+#
+
+#</IfModule>
+# End of proxy directives.
+
+### Section 3: Virtual Hosts
+#
+# VirtualHost: If you want to maintain multiple domains/hostnames on your
+# machine you can setup VirtualHost containers for them. Most configurations
+# use only name-based virtual hosts so the server doesn't need to worry about
+# IP addresses. This is indicated by the asterisks in the directives below.
+#
+# Please see the documentation at 
+# <URL:http://httpd.apache.org/docs/2.2/vhosts/>
+# for further details before you try to setup virtual hosts.
+#
+# You may use the command line option '-S' to verify your virtual host
+# configuration.
+
+#
+# Use name-based virtual hosting.
+#
+#NameVirtualHost *:80
+#
+# NOTE: NameVirtualHost cannot be used without a port specifier 
+# (e.g. :80) if mod_ssl is being used, due to the nature of the
+# SSL protocol.
+#
+
+#
+# VirtualHost example:
+# Almost any Apache directive may go into a VirtualHost container.
+# The first VirtualHost section is used for requests without a known
+# server name.
+#
+#<VirtualHost *:80>
+#    ServerAdmin webmaster@dummy-host.example.com
+#    DocumentRoot /www/docs/dummy-host.example.com
+#    ServerName dummy-host.example.com
+#    ErrorLog logs/dummy-host.example.com-error_log
+#    CustomLog logs/dummy-host.example.com-access_log common
+#</VirtualHost>
+
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgweb/lockfile
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+check_lock() {
+    if [ -e "$1" ]
+    then
+        return 1
+    else
+        touch "$1"
+        return 0
+    fi
+}
+
+RETRIES=15;
+
+if [ "$1" = "-r" ]; then
+    RETRIES="$2"
+    shift 2
+fi
+
+LOCKFILE=$1
+
+if [ -z "$1" ]; then
+    echo "Filename required"
+    exit 1
+fi
+
+while [ $RETRIES -ne 0 ]; do
+    if check_lock $LOCKFILE
+    then
+        exit 0
+    else
+        echo "Repository locked, will try again in 8 seconds"
+        sleep 8
+        RETRIES=$(expr $RETRIES - 1)
+    fi
+done
+
+echo "Failed to grab lock file $LOCKFILE, did a push fail and leave a stale lockfile?"
+
+exit 1
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgweb/mirror-pull
@@ -0,0 +1,169 @@
+#!/bin/sh
+
+die() {
+    echo "$1" 1>&2
+    exit ${2:-1}
+}
+
+usage() {
+    echo "USAGE"
+    echo " $0 [-r source_repo_root] [-t target_directory] [-c changeset] [-d] -e <repository>"
+    echo
+}
+
+check_nowait_lock() {
+    if [ -e $1 ]; then
+        return 1
+    fi
+
+    return 0
+}
+
+check_wait_lock() {
+    if [ -e $1 ]; then
+        sleep 5
+        check_wait_lock $1
+    fi
+    return 0
+}
+
+create_lock() {
+    check_nowait_lock $1 && touch $1
+}
+
+if [ -n "$SSH_ORIGINAL_COMMAND" ]; then
+    set -- $SSH_ORIGINAL_COMMAND
+fi
+
+TEMP=`getopt --options hdr:t:l:c:e: --long help,delete,hgrc,root:,target:lock-retries:changeset:repo: \
+    -n 'mirror-pull' -- "$@"`
+if [ $? != 0 ]; then
+    die "getopt barfed"
+fi
+
+eval set -- "$TEMP"
+
+# defaults:
+REPO_ROOT=ssh://<%= @mirror_source %>
+REPO_TARGET=<%= @repo_serve_path %>
+LOCK_RETRIES=3 # 3 retries, the first is 5, second is 10, third is 15
+DELETE=false
+HGRC=false
+CHANGESET=""
+
+export PYTHONPATH=<%= @python_lib_override_path %>:<%= @python_lib_path %>
+
+while true; do
+    case "$1" in 
+        -r|--root)
+            REPO_ROOT="$2" ; shift 2
+            ;;
+        -t|--target)
+            REPO_TARGET="$2" ; shift 2
+            ;;
+        -l|--lock-retries)
+            LOCK_RETRIES="$2" ; shift 2
+            ;;
+        -c|--changeset)
+            CHANGESET="-r $2" ; shift 2
+            ;;
+        -e|--repo)
+            REPO="$2" ; shift 2
+            ;;
+        -d|--delete)
+            DELETE=true ; shift 1
+            ;;
+        --hgrc)
+            HGRC=true ; shift 1
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        --)
+            shift ; break
+            ;;
+        *)
+            die "I don't know what happened."
+            ;;
+    esac
+done
+
+if [ -n "$REPO" ]; then
+    repo=$REPO
+else
+    repo="$1"
+fi
+
+repo=`echo "$repo" | sed 's#[^-\._/[:alnum:]]##g'`
+test -z "$repo" && die "need a repo to clone, relative to %<= @repo_serve_path %>" 
+
+name=`echo $repo | sed 's#^/*##'`
+src=${REPO_ROOT}/$name
+
+GLOBAL_HG_OPTS="--config hooks.pretxnchangegroup.z_linearhistory= --config hooks.pretxnchangegroup.z_loghistory= --config trusted.users=root,hg --config paths.default=ssh://<%= @mirror_source %>/${name}"
+
+if [ "$DELETE" = "true" ]; then
+    if [ "$(echo ${repo}|cut -f1 --delimiter=/)" != "users" ]; then
+        echo "You requested to delete ${repo}. This does not begin with 'users/', ignoring."
+        exit 1
+    fi
+    if [ ! -d $REPO_TARGET/$repo ]; then
+        echo "No repo found at $repo"
+        exit 1
+    fi
+    echo "Deleting repo $repo"
+    /bin/rm -rf $REPO_TARGET/$repo
+    exit 0
+fi
+
+cd $REPO_TARGET || die "$REPO_TARGET does not exist, cannot create repositories there"
+
+retry() {
+    local _cmd=$*
+    eval $_cmd
+    res=$?
+    if [ "$res" != 0 ]; then
+        sleep 1
+        echo "retrying $_cmd"
+        eval $_cmd
+        res=$?
+    fi
+    return $res
+}
+
+if [ "$HGRC" = "true" ]; then
+    if [ ! -d $repo/.hg ]; then
+        echo "No repo found at $repo"
+        exit 1
+    fi
+    echo "Updating hgrc for $repo"
+    TEMPFILE=$(/bin/mktemp)
+    retry "ssh -i <%= @mirror_priv_key_path %> -l <%= @mirror_user_name %> <%= @mirror_source %> repo-config $name > $TEMPFILE" && /bin/mv $TEMPFILE $repo/.hg/hgrc
+    exit 0
+fi
+
+exit_code=0
+if [ -d "$name" ]; then
+    echo "$name already exists, pulling"
+    retry hg -R $name pull $GLOBAL_HG_OPTS $CHANGESET
+    exit_code=$?
+elif [ \! -e $name ]; then
+    mkdir -p `dirname $name` 
+    echo "$name does not yet exist, cloning"
+    lockfile -r $LOCK_RETRIES ${PWD}/syncing.lck || die "Could not lock destination"
+    retry hg clone $GLOBAL_HG_OPTS -U -v "$src" $name
+    exit_code=$?
+    rm ${PWD}/syncing.lck
+else
+    die "WTF is $REPO_TARGET/$name"
+fi
+
+exit $exit_code
+
+# Local variables:
+# mode: shell-script
+# tab-width: 4
+# indent-tabs-mode: nil
+# end:
+
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/mod_wsgi.conf
@@ -0,0 +1,1 @@
+LoadModule wsgi_module modules/mod_wsgi.so
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/mozilla.repo
@@ -0,0 +1,5 @@
+[mozilla]
+name=Mozilla Package Repo - $basearch
+baseurl=https://mrepo.mozilla.org/mrepo/$releasever-$basearch/RPMS.mozilla
+enabled=1
+gpgcheck=0
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgweb/repo-group
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+import cgi
+import cgitb
+import sys
+import re
+import subprocess
+
+#cgitb.enable()
+cgitb.enable(display=0)
+
+
+def check_repo_name(repo_name):
+    good_chars = re.compile('^(\w|-|/|\.\w)+\s*$')
+    if not good_chars.search(repo_name):
+        sys.stderr.write('Only alpha-numeric characters, ".", and "-" are '
+                         'allowed in the repository names.\n')
+        sys.stderr.write('Please try again with only those characters.\n')
+        sys.exit(1)
+    return True
+
+print "Content-Type: text/plain"
+print
+
+form = cgi.FieldStorage()
+#if not form or not form.has_key("repo"):
+if not form or "repo" not in form:
+    print "Need a repository to check"
+    sys.exit(1)
+
+repo = form.getfirst("repo", "")
+if not check_repo_name(repo):
+    print "You've included some illegal characters in your repo name"
+    sys.stderr.write("Warning: illegal characters in repo name\n")
+    sys.exit(1)
+# ensure that the repo is within repo_root
+if repo.find('/../') != -1:
+    print "That's not allowed"
+    sys.stderr.write("Warning: /../ found in a repo name.\n")
+    sys.exit(1)
+
+command = ('/usr/bin/ssh -o StrictHostKeyChecking=no -o '
+           'UserKnownHostsFile=/dev/null -l vcs-sync@mozilla.com '
+           '-i /etc/mercurial/mirror hg.mozilla.org '
+           'repo-group %s' % repo)
+proc = subprocess.Popen(command,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        shell=True)
+(out, err) = proc.communicate()
+print out
+
+# Local variables:
+# mode: python
+# indent-tabs-mode: nil
+# tab-width: 4
+# end:
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgweb/run.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+/sbin/service rsyslog start
+/usr/sbin/httpd -DFOREGROUND &
+/usr/sbin/sshd -D &
+wait
new file mode 100755
--- /dev/null
+++ b/testing/docker/builder-hgweb/set-mirror-key.py
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import grp
+import os
+import pwd
+import stat
+import sys
+
+priv, pub = sys.argv[1:]
+
+with open('/etc/mercurial/mirror', 'wb') as fh:
+    fh.write(priv)
+
+with open('/etc/mercurial/mirror.pub', 'wb') as fh:
+    fh.write(pub)
+
+os.chmod('/etc/mercurial/mirror', stat.S_IRUSR | stat.S_IWUSR)
+os.chmod('/etc/mercurial/mirror.pub', stat.S_IRUSR | stat.S_IWUSR)
+
+uhg = pwd.getpwnam('hg')
+ghg = grp.getgrnam('hg')
+
+os.chown('/etc/mercurial/mirror', uhg.pw_uid, ghg.gr_gid)
+os.chown('/etc/mercurial/mirror.pub', uhg.pw_uid, ghg.gr_gid)
+
+# Allow SSH connections from the master server.
+with open('/home/hg/.ssh/authorized_keys', 'wb') as fh:
+    fh.write('command="/usr/local/bin/mirror-pull -t /repo_local/mozilla/mozilla $SSH_ORIGINAL_COMMAND"')
+    fh.write(',no-pty,no-x11-forwarding,no-agent-forwarding ')
+    fh.write(pub.strip())
+    fh.write('\n')
+
+os.chown('/home/hg/.ssh/authorized_keys', uhg.pw_uid, ghg.gr_gid)
+os.chmod('/home/hg/.ssh/authorized_keys', stat.S_IRUSR | stat.S_IWUSR)
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/ssh-config
@@ -0,0 +1,12 @@
+Host hg.mozilla.org
+    User vcs-sync@mozilla.com
+    Hostname hg.mozilla.org
+    IdentityFile /etc/mercurial/mirror
+    ServerAliveCountMax 3
+    ServerAliveInterval 5
+    ConnectionAttempts 3
+    ConnectTimeout 15
+# Bug 1030759
+#    ControlMaster auto
+#    ControlPath ~/.ssh/%r@%h:%p
+
new file mode 100644
--- /dev/null
+++ b/testing/docker/builder-hgweb/vhost.conf
@@ -0,0 +1,28 @@
+# vim: syntax=apache
+
+WSGISocketPrefix /var/run/wsgi
+<VirtualHost *:80>
+    ServerName hg
+    DocumentRoot /repo_local/mozilla/webroot_wsgi
+
+    SetEnv HGENCODING UTF-8
+    SetEnv LC_TYPE UTF-8
+
+    WSGIScriptAlias / /repo_local/mozilla/webroot_wsgi/hgweb.wsgi
+
+    WSGIDaemonProcess hg processes=48 threads=1 maximum-requests=20 deadlock-timeout=60 inactivity-timeout=300 user=hg group=hg display-name=hg
+    WSGIProcessGroup hg
+
+    AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css application/json
+
+    #LogLevel debug
+    LogFormat "%h %v %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\""
+    ErrorLog "/var/log/httpd/hg/error_log"
+    CustomLog "/var/log/httpd/hg/access_log" combined env=!image
+</VirtualHost>
+
+# Local variables:
+# mode: apache
+# tab-width: 4
+# indent-tabs-mode: nil
+# end:
--- a/testing/vcttesting/docker.py
+++ b/testing/vcttesting/docker.py
@@ -80,31 +80,41 @@ class Docker(object):
             'last-pulse-id': None,
             'last-web-id': None,
             'last-rbweb-id': None,
             'last-db-bootstrap-id': None,
             'last-web-bootstrap-id': None,
             'last-rbweb-bootstrap-id': None,
             'last-autolanddb-id': None,
             'last-autoland-id': None,
+            'last-hgmaster-id': None,
+            'last-hgweb-id': None,
+            'last-ldap-id': None,
         }
 
         if os.path.exists(state_path):
             with open(state_path, 'rb') as fh:
                 self.state = json.load(fh)
 
-        self.state.setdefault('last-db-id', None)
-        self.state.setdefault('last-pulse-id', None)
-        self.state.setdefault('last-web-id', None)
-        self.state.setdefault('last-rbweb-id', None)
-        self.state.setdefault('last-db-bootstrap-id', None)
-        self.state.setdefault('last-web-bootstrap-id', None)
-        self.state.setdefault('last-rbweb-bootstrap-id', None)
-        self.state.setdefault('last-autolanddb-id', None)
-        self.state.setdefault('last-autoland-id', None)
+        keys = (
+            'last-db-id',
+            'last-pulse-id',
+            'last-web-id',
+            'last-rbweb-id',
+            'last-db-bootstrap-id',
+            'last-web-bootstrap-id',
+            'last-rbweb-bootstrap-id',
+            'last-autolanddb-id',
+            'last-autoland-id',
+            'last-hgmaster-id',
+            'last-hgweb-id',
+            'last-ldap-id',
+        )
+        for k in keys:
+            self.state.setdefault(k, None)
 
         self.client = docker.Client(base_url=url, tls=tls)
 
         # Try to obtain a network hostname for the Docker server. We use this
         # for determining where to look for opened ports.
         # This is a bit complicated because Docker can be running from a local
         # socket or or another host via something like boot2docker.
         # TODO look at network info for Docker and extract IP address instead.
@@ -247,27 +257,37 @@ class Docker(object):
 
     def build_hgmo(self, verbose=False):
         """Ensure the images for a hg.mozilla.org service are built.
 
         hg-master runs the ssh service while hg-slave runs hgweb. The mirroring
         and other bits should be the same as in production with the caveat that
         LDAP integration is probably out of scope.
         """
-        with futures.ThreadPoolExecutor(2) as e:
+        with futures.ThreadPoolExecutor(3) as e:
             f_hg = e.submit(self.ensure_built, 'hgmaster', add_vct=True,
                             verbose=verbose)
+            f_hgweb = e.submit(self.ensure_built, 'hgweb', add_vct=True,
+                               verbose=verbose)
             f_ldap = e.submit(self.ensure_built, 'ldap', verbose=verbose)
 
         hg_master_image = f_hg.result()
+        hgweb_image = f_hgweb.result()
         ldap_image = f_ldap.result()
 
         self.state['last-hgmaster-id'] = hg_master_image
+        self.state['last-hgweb-id'] = hgweb_image
         self.state['last-ldap-id'] = ldap_image
 
+        return {
+            'hgmaster': hg_master_image,
+            'hgweb': hgweb_image,
+            'ldap': ldap_image,
+        }
+
     def build_mozreview(self, verbose=False):
         """Ensure the images for a MozReview service are built.
 
         bmoweb's entrypoint does a lot of setup on first run. This takes many
         seconds to perform and this cost is unacceptable for efficient test
         running. So, when we build the BMO images, we create throwaway
         containers and commit the results to a new image. This allows us to
         spin up multiple bmoweb containers very quickly.
@@ -636,30 +656,36 @@ class Docker(object):
             self.state['last-pulse-id'],
             self.state['last-web-id'],
             self.state['last-rbweb-id'],
             self.state['last-db-bootstrap-id'],
             self.state['last-web-bootstrap-id'],
             self.state['last-rbweb-bootstrap-id'],
             self.state['last-autolanddb-bootstrap-id'],
             self.state['last-autoland-bootstrap-id'],
+            self.state['last-hgmaster-id'],
+            self.state['last-hgweb-id'],
+            self.state['last-ldap-id'],
         ])
 
         relevant_repos = set([
             'bmoweb',
             'bmoweb-bootstrapped',
             'bmodb-volatile',
             'bmodb-volatile-bootstrapped',
             'pulse',
             'rbweb',
             'rbweb-bootstrapped',
             'autolanddb',
             'autolanddb-bootstrapped',
             'autoland',
             'autoland-bootstrapped',
+            'hgmaster',
+            'hgweb',
+            'ldap',
         ])
 
         for i in self.client.images():
             iid = i['Id']
 
             # Don't do anything with images attached to running containers -
             # Docker won't allow it.
             if iid in running:
@@ -703,8 +729,17 @@ class Docker(object):
         immediately after it is started.
         """
         self.client.start(cid, **kwargs)
         try:
             state = self.client.inspect_container(cid)
             yield state
         finally:
             self.client.stop(cid, timeout=20)
+
+    def get_file_content(self, cid, path):
+        """Get the contents of a file from a container."""
+        r = self.client.copy(cid, path)
+        buf = BytesIO(r.read())
+        buf.seek(0)
+        t = tarfile.open(mode='r', fileobj=buf)
+        fp = t.extractfile(os.path.basename(path))
+        return fp.read()
new file mode 100644
--- /dev/null
+++ b/testing/vcttesting/hgmo.py
@@ -0,0 +1,265 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import errno
+import json
+import os
+
+import concurrent.futures as futures
+import ldap
+import paramiko
+
+
+class HgCluster(object):
+    """Interface to a cluster of HG servers.
+
+    This class manages Docker contains and environments that replicate the
+    hg.mozilla.org server configuration.
+    """
+
+    def __init__(self, docker, state_path=None):
+        self._d = docker
+        self._dc = docker.client
+        self.state_path = state_path
+
+        if state_path and os.path.exists(state_path):
+            with open(state_path, 'rb') as fh:
+                state = json.load(fh)
+                for k, v in state.items():
+                    setattr(self, k, v)
+        else:
+            self.ldap_image = None
+            self.master_image = None
+            self.ldap_id = None
+            self.master_id = None
+            self.web_ids = []
+            self.ldap_uri = None
+            self.master_ssh_hostname = None
+            self.master_ssh_port = None
+            self.web_urls = []
+
+    def start(self, ldap_port=None, master_ssh_port=None, web_count=2):
+        """Start the cluster."""
+
+        images = self._d.build_hgmo(verbose=True)
+
+        master_image = images['hgmaster']
+        web_image = images['hgweb']
+        ldap_image = images['ldap']
+
+        with futures.ThreadPoolExecutor(4) as e:
+            f_ldap_create = e.submit(self._dc.create_container, ldap_image)
+            f_master_create = e.submit(self._dc.create_container,
+                                       master_image)
+            f_web_creates = []
+            for i in range(web_count):
+                f_web_creates.append(e.submit(self._dc.create_container,
+                                              web_image))
+
+            ldap_id = f_ldap_create.result()['Id']
+            master_id = f_master_create.result()['Id']
+
+            # Start LDAP first because we need to link it to all the hg
+            # containers.
+            self._dc.start(ldap_id, port_bindings={389: ldap_port})
+            ldap_state = self._dc.inspect_container(ldap_id)
+
+            self._dc.start(master_id,
+                           links=[(ldap_state['Name'], 'ldap')],
+                           port_bindings={22: master_ssh_port})
+
+            master_state = self._dc.inspect_container(master_id)
+
+            web_ids = [f.result()['Id'] for f in f_web_creates]
+            fs = []
+            for i in web_ids:
+                fs.append(e.submit(self._dc.start, i,
+                                   links=[(master_state['Name'], 'master')],
+                                   port_bindings={22: None, 80: None}))
+            [f.result() for f in fs]
+
+            f_web_states = []
+            for i in web_ids:
+                f_web_states.append(e.submit(self._dc.inspect_container, i))
+
+            web_states = [f.result() for f in f_web_states]
+
+        with futures.ThreadPoolExecutor(4) as e:
+            f_private_key = e.submit(self._d.get_file_content, master_id, '/etc/mercurial/mirror')
+            f_public_key = e.submit(self._d.get_file_content, master_id, '/etc/mercurial/mirror.pub')
+
+        mirror_private_key = f_private_key.result()
+        mirror_public_key = f_public_key.result()
+
+        # Reconcile state across all the containers.
+        with futures.ThreadPoolExecutor(web_count + 1) as e:
+            # Update the SSH key for the "hg" user on the web containers.
+            cmd = [
+                '/set-mirror-key.py',
+                mirror_private_key,
+                mirror_public_key,
+            ]
+            for i in web_ids:
+                e.submit(self._dc.execute(i, cmd))
+
+            # Tell the master about all the mirrors.
+            mirrors = [s['NetworkSettings']['IPAddress'] for s in web_states]
+            e.submit(self._dc.execute, master_id, ['/set-mirrors.py'] + mirrors)
+
+        ldap_ports = ldap_state['NetworkSettings']['Ports']
+        master_ssh_ports = master_state['NetworkSettings']['Ports']
+
+        ldap_host_port = ldap_ports['389/tcp'][0]['HostPort']
+        master_ssh_host_port = master_ssh_ports['22/tcp'][0]['HostPort']
+
+        self.ldap_image = ldap_image
+        self.master_image = master_image
+        self.web_image = web_image
+        self.ldap_id = ldap_id
+        self.master_id = master_id
+        self.web_ids = web_ids
+        self.ldap_uri = 'ldap://%s:%s/' % (self._d.docker_hostname,
+                                           ldap_host_port)
+        self.master_ssh_hostname = self._d.docker_hostname
+        self.master_ssh_port = master_ssh_host_port
+        self.web_urls = []
+        for s in web_states:
+            port = s['NetworkSettings']['Ports']['80/tcp'][0]['HostPort']
+            self.web_urls.append('http://%s:%s/' % (self._d.docker_hostname,
+                                                    port))
+
+        return self._write_state()
+
+    def stop(self):
+        """Stop the cluster.
+
+        Containers will be shut down gracefully.
+        """
+        c = self._d.client
+        with futures.ThreadPoolExecutor(4) as e:
+            e.submit(c.stop, self.master_id)
+            e.submit(c.stop, self.ldap_id)
+            for i in self.web_ids:
+                e.submit(c.stop, i)
+
+    def clean(self):
+        """Clean the cluster.
+
+        Containers will be shut down and removed. The state file will
+        destroyed.
+        """
+        self.stop()
+        c = self._d.client
+        with futures.ThreadPoolExecutor(4) as e:
+            e.submit(c.remove_container, self.master_id)
+            e.submit(c.remove_container, self.ldap_id)
+            for i in self.web_ids:
+                e.submit(c.remove_container, i)
+
+        try:
+            os.unlink(self.state_path)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+        self.ldap_id = None
+        self.master_id = None
+        self.web_ids = []
+        self.ldap_uri = None
+        self.master_ssh_hostname = None
+        self.master_ssh_port = None
+
+    def _write_state(self):
+        assert self.state_path
+        s = {
+                'ldap_image': self.ldap_image,
+                'master_image': self.master_image,
+                'web_image': self.web_image,
+                'ldap_id': self.ldap_id,
+                'master_id': self.master_id,
+                'web_ids': self.web_ids,
+                'ldap_uri': self.ldap_uri,
+                'master_ssh_hostname': self.master_ssh_hostname,
+                'master_ssh_port': self.master_ssh_port,
+                'web_urls': self.web_urls,
+        }
+        with open(self.state_path, 'wb') as fh:
+            json.dump(s, fh, sort_keys=True, indent=4)
+
+        return s
+
+    @property
+    def ldap(self):
+        assert self.ldap_uri
+        c = ldap.initialize(self.ldap_uri)
+        c.simple_bind_s('cn=admin,dc=mozilla', 'password')
+
+        return c
+
+    def create_ldap_user(self, email, username, uid, fullname,
+                         key_filename=None):
+        """Create a new user in LDAP.
+
+        The user has an ``email`` address, a full ``name``, a
+        ``username`` (for system accounts) and a numeric ``uid``.
+        """
+
+        dn = 'mail=%s,o=com,dc=mozilla' % email
+
+        r = [
+            (b'objectClass', [
+                b'hgAccount',
+                b'inetOrgPerson',
+                b'organizationalPerson',
+                b'person',
+                b'posixAccount',
+                b'top',
+            ]),
+            (b'cn', [fullname]),
+            (b'fakeHome', [b'/tmp']),
+            (b'gidNumber', [b'100']),
+            (b'hgAccountEnabled', [b'TRUE']),
+            (b'hgHome', [b'/tmp']),
+            (b'hgShell', [b'/bin/sh']),
+            (b'homeDirectory', [b'/home/%s' % username]),
+            (b'sn', [fullname.split()[-1]]),
+            (b'uid', [username]),
+            (b'uidNumber', [str(uid)]),
+        ]
+
+        self.ldap.add_s(dn, r)
+
+        if key_filename:
+            pubkey_filename = '%s.pub' % key_filename
+            if os.path.exists(key_filename):
+                with open(pubkey_filename, 'rb') as fh:
+                    pubkey = fh.read()
+            else:
+                k = paramiko.rsakey.RSAKey.generate(2048)
+                k.write_private_key_file(key_filename)
+                pubkey = b'%s %s %s' % (k.get_name(), k.get_base64(), email)
+                with open(pubkey_filename, 'wb') as fh:
+                    fh.write(pubkey)
+
+            self.add_ssh_key(email, pubkey)
+
+    def add_ssh_key(self, email, key):
+        """Add an SSH key to a user in LDAP."""
+        dn = 'mail=%s,o=com,dc=mozilla' % email
+
+        c = self.ldap
+        modlist = []
+
+        try:
+            existing = c.search_s(dn, ldap.SCOPE_BASE)[0][1]
+            if b'ldapPublicKey' not in existing[b'objectClass']:
+                modlist.append((ldap.MOD_ADD, b'objectClass', b'ldapPublicKey'))
+        except ldap.NO_SUCH_OBJECT:
+            pass
+
+        modlist.append((ldap.MOD_ADD, b'sshPublicKey', key))
+
+        c.modify_s(dn, modlist)
new file mode 100644
--- /dev/null
+++ b/testing/vcttesting/hgmo_mach_commands.py
@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import sys
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+
+@CommandProvider
+class HgmoCommands(object):
+    def __init__(self, context):
+        from vcttesting.docker import Docker, params_from_env
+        from vcttesting.hgmo import HgCluster
+
+        if 'HGMO_STATE_FILE' not in os.environ:
+            print('Do not know where to store state.')
+            print('Set the HGMO_STATE_FILE environment variable and try again.')
+            sys.exit(1)
+
+        if 'DOCKER_STATE_FILE' not in os.environ:
+            print('Do not where to store Docker state.')
+            print('Set the DOCKER_STATE_FILE environment variable and try again.')
+            sys.exit(1)
+
+        docker_url, tls = params_from_env(os.environ)
+        if not docker_url:
+            print('Docker not available')
+            sys.exit(1)
+
+        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
+        self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'])
+
+    @Command('start', category='hgmo',
+             description='Start a hg.mozilla.org cluster')
+    def start(self):
+        s = self.c.start()
+        print('SSH Hostname: %s' % s['master_ssh_hostname'])
+        print('SSH Port: %s' % s['master_ssh_port'])
+        print('LDAP URI: %s' % s['ldap_uri'])
+        for url in s['web_urls']:
+            print('Web URL: %s' % url)
+
+    @Command('stop', category='hgmo',
+             description='Stop a hg.mozilla.org cluster')
+    def stop(self):
+        self.c.stop()
+
+    @Command('clean', category='hgmo',
+             description='Clean up all references to this cluster')
+    def clean(self):
+        self.c.clean()
+
+    @Command('create-ldap-user', category='hgmo',
+             description='Create a new user in LDAP')
+    @CommandArgument('email',
+                     help='Email address associated with user')
+    @CommandArgument('username',
+                     help='System account name')
+    @CommandArgument('uid', type=int,
+                     help='Numeric user ID to associate with user')
+    @CommandArgument('fullname',
+                     help='Full name of the user')
+    @CommandArgument('--key-file',
+                     help='Use or create an SSH key')
+    def create_ldap_user(self, email, username, uid, fullname, key_file=None):
+        self.c.create_ldap_user(email, username, uid, fullname,
+                                key_filename=key_file)
+
+    @Command('add-ssh-key', category='hgmo',
+             description='Add an SSH public key to a user')
+    @CommandArgument('email',
+                     help='Email address of user to modify')
+    @CommandArgument('key',
+                     help='SSH public key string')
+    def add_ssh_key(self, email, key):
+        if key == '-':
+            key = sys.stdin.read().strip().encode('utf-8')
+        self.c.add_ssh_key(email, key)