hgmo: optimize Docker image generation
authorGregory Szorc <gps@mozilla.com>
Tue, 10 Mar 2015 11:45:51 -0700
changeset 360525 cccdde2d9f45aff35ac583701de64f8567c0adfc
parent 360524 2931ea56d7c4b5a4fc901d87ff3a181d3633196c
child 360526 fe2c0bba8fa52adc0ad682020e89b917af967a2f
push id16998
push userrwood@mozilla.com
push dateMon, 02 May 2016 19:42:03 +0000
hgmo: optimize Docker image generation We use the new feature to limit which files are added to the context. We also consolidate the number of instructions in the Dockerfiles to make image building faster. The images still don't build as quickly as I would like. We may have to move instructions into a shell script that is executed inside the container.
testing/docker/builder-hgmaster/Dockerfile
testing/docker/builder-hgweb/Dockerfile
testing/vcttesting/docker.py
--- a/testing/docker/builder-hgmaster/Dockerfile
+++ b/testing/docker/builder-hgmaster/Dockerfile
@@ -21,59 +21,52 @@ RUN curl https://mrepo.mozilla.org/mrepo
 RUN yum install -y authconfig nss-pam-ldapd openldap-clients pam_ldap python-ldap sudo syslog
 RUN yum install -y /openssh-lpk.rpm /openssh-lpk-clients.rpm /openssh-lpk-server.rpm
 
 RUN yum localinstall -y https://mrepo.mozilla.org/mrepo/6Server-x86_64/RPMS.mozilla/mercurial-3.3.2-0.x86_64.rpm
 
 RUN yum clean all
 
 # These should stay in sync with mozilla.ldif from the ldap server.
-RUN groupadd -g 600 scm_level_1
-RUN groupadd -g 601 scm_level_2
-RUN groupadd -g 602 scm_level_3
+RUN groupadd -g 600 scm_level_1 && groupadd -g 601 scm_level_2 && groupadd -g 603 scm_level_3
 
 # Add the hg user
 RUN useradd hg
 
 # Set root password to something for debugging purposes
 RUN echo root:mozillafirefox|chpasswd
 
 # Set up Mercurial, adding example content
 RUN mkdir -p /etc/mercurial
 RUN mkdir -p /repo/hg/mozilla /repo/hg/extensions /repo/hg/scripts
 
 # Set up users directories, make permissions right
 RUN mkdir -p /repo/hg/mozilla/users
-RUN chmod 2775 /repo/hg/mozilla/users
-RUN chown -R hg:scm_level_1 /repo/hg/mozilla/users
+RUN chmod 2775 /repo/hg/mozilla/users && chown -R hg:scm_level_1 /repo/hg/mozilla/users
 
 # TODO: Fix this logging or remove record-pushes script as it's information duplicates serverlog
-RUN touch /var/log/hg-push.log; chmod 666 /var/log/hg-push.log
+RUN touch /var/log/hg-push.log && chmod 666 /var/log/hg-push.log
 
 RUN touch /var/log/pash.log && chmod 666 /var/log/pash.log
 
 ADD sshd_config /etc/ssh/sshd_config
 ADD nslcd.conf /etc/nslcd.conf
 
 RUN ssh-keygen -b 2048 -f /etc/mercurial/mirror -t rsa
 RUN mkdir /home/hg/.ssh
 RUN cp /etc/mercurial/mirror.pub /home/hg/.ssh/authorized_keys
 RUN chown -R hg:hg /home/hg/.ssh
 RUN chmod 640 /home/hg/.ssh/authorized_keys
 
-ADD create-repo /create-repo
-ADD set-mirrors.py /set-mirrors.py
-ADD entrypoint.py /entrypoint.py
+ADD create-repo set-mirrors.py entrypoint.py /
 
-ADD extra/vct/testing/docker/builder-hgmaster/hgrc /etc/mercurial/hgrc
-ADD extra/vct/scripts/pash/* /usr/local/bin/
-ADD extra/vct/hgext/pushlog-legacy/* /repo/hg/extensions/
+ADD hgrc /etc/mercurial/hgrc
+ADD extra/vct/scripts/pash/* extra/vct/scripts/repo-push.sh /usr/local/bin/
+ADD extra/vct/hgext/pushlog-legacy/buglink.py extra/vct/hgext/pushlog-legacy/feedparser.py extra/vct/hgext/pushlog-legacy/hgwebjson.py extra/vct/hgext/pushlog-legacy/pushlog-feed.py /repo/hg/extensions/
 ADD extra/vct/hgext/pushlog-legacy/parsedatetime/* /repo/hg/extensions/parsedatetime/
 ADD extra/vct/hgext/serverlog/* /repo/hg/extensions/serverlog/
 ADD extra/vct/hghooks /repo/hg/libraries/
-ADD extra/vct/scripts/record-pushes.sh /repo/hg/scripts/
-ADD extra/vct/scripts/push-repo.sh /repo/hg/scripts/
-ADD extra/vct/scripts/repo-push.sh /usr/local/bin/
+ADD extra/vct/scripts/record-pushes.sh extra/vct/scripts/push-repo.sh /repo/hg/scripts/
 
 EXPOSE 22
 
 ENTRYPOINT ["/entrypoint.py"]
 CMD ["/usr/sbin/sshd", "-D"]
--- a/testing/docker/builder-hgweb/Dockerfile
+++ b/testing/docker/builder-hgweb/Dockerfile
@@ -22,32 +22,26 @@ RUN yum clean all
 RUN hg clone http://hg.mozilla.org/hgcustom/library_overrides /repo_local/mozilla/library_overrides
 
 ADD httpd.conf /etc/httpd/conf/httpd.conf
 ADD vhost.conf /etc/httpd/conf.d/hg-vhost.conf
 ADD mod_wsgi.conf /etc/httpd/conf.d/wsgi.conf
 RUN mkdir /var/log/httpd/hg
 
 ADD hgrc /etc/mercurial/hgrc
-ADD mirror-pull /usr/local/bin/mirror-pull
-ADD repo-group /usr/local/bin/repo-group
-ADD lockfile /usr/local/bin/lockfile
+ADD mirror-pull repo-group lockfile /usr/local/bin/
 ADD ssh-config /home/hg/.ssh/config
 
 ADD extra/vct/hgtemplates /repo_local/mozilla/hg_templates
 ADD extra/vct/hghooks /repo_local/mozilla/hghooks
 
-ADD extra/vct/hgext/pushlog-legacy/buglink.py /repo_local/mozilla/extensions
-ADD extra/vct/hgext/pushlog-legacy/feedparser.py /repo_local/mozilla/extensions
-ADD extra/vct/hgext/pushlog-legacy/hgwebjson.py /repo_local/mozilla/extensions
-ADD extra/vct/hgext/pushlog-legacy/pushlog-feed.py /repo_local/mozilla/extensions
+ADD extra/vct/hgext/pushlog-legacy/*.py /repo_local/mozilla/extensions/
 RUN ln -sf /repo_local/mozilla/hghooks/mozhghooks /repo_local/mozilla/libraries
 
-ADD hgweb.wsgi /repo_local/mozilla/webroot_wsgi/hgweb.wsgi
-ADD hgweb.config /repo_local/mozilla/webroot_wsgi/hgweb.config
+ADD hgweb.wsgi hgweb.config /repo_local/mozilla/webroot_wsgi/
 ADD set-mirror-key.py /set-mirror-key.py
 ADD entrypoint.py /entrypoint.py
 ADD run.sh /run.sh
 
 EXPOSE 22
 EXPOSE 80
 
 ENTRYPOINT ["/entrypoint.py"]
--- a/testing/vcttesting/docker.py
+++ b/testing/vcttesting/docker.py
@@ -288,19 +288,33 @@ class Docker(object):
     def build_hgmo(self, verbose=False):
         """Ensure the images for a hg.mozilla.org service are built.
 
         hg-master runs the ssh service while hg-slave runs hgweb. The mirroring
         and other bits should be the same as in production with the caveat that
         LDAP integration is probably out of scope.
         """
         with futures.ThreadPoolExecutor(3) as e:
-            f_hg = e.submit(self.ensure_built, 'hgmaster', add_vct=True,
+            master_paths = (
+                'scripts/pash/',
+                'hgext/pushlog-legacy/',
+                'hgext/serverlog/',
+                'hghooks/',
+                'scripts/',
+            )
+            f_hg = e.submit(self.ensure_built, 'hgmaster',
+                            vct_paths=master_paths,
                             verbose=verbose)
-            f_hgweb = e.submit(self.ensure_built, 'hgweb', add_vct=True,
+            web_paths = (
+                'hgtemplates/',
+                'hghooks/',
+                'hgext/pushlog-legacy/',
+            )
+            f_hgweb = e.submit(self.ensure_built, 'hgweb',
+                               vct_paths=web_paths,
                                verbose=verbose)
             f_ldap = e.submit(self.ensure_built, 'ldap', verbose=verbose)
 
         hg_master_image = f_hg.result()
         hgweb_image = f_hgweb.result()
         ldap_image = f_ldap.result()
 
         self.state['last-hgmaster-id'] = hg_master_image