ansible: use stream clones in project repo reset playbook (Bug 1672877) r=zeid
authorConnor Sheehan <sheehan@mozilla.com>
Wed, 28 Oct 2020 17:30:31 +0000
changeset 7630 ddcc57002b724274017e139801c0e3e72444b9f8
parent 7629 bf3e7c8155eb926098809b04d9e0963dab83c923
child 7631 41809c2415d035efe933ca0edff833c2e45a4836
push id3807
push usercosheehan@mozilla.com
push dateWed, 28 Oct 2020 17:35:02 +0000
treeherderversion-control-tools@ddcc57002b72 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerszeid
bugs1672877, 1672465
ansible: use stream clones in project repo reset playbook (Bug 1672877) r=zeid Currently when we reset a twig (project branch) on hgmo by running the Ansible playbook `hgmo-reset-twig.yml`, the repo is reset by creating a new empty repo on hgssh and locally pulling from `mozilla-central`. This creates an `hg-repo-init-2` message in vcsreplicator so the repo is created on hgweb, after which we run `hg replicatesync` to trigger a full sync of the repo. This full sync causes all hgweb nodes to pull from hgssh to gather the new data. This process is slow, and causes the vcsreplicator queue for the project repos to lag far behind. This triggers many alerts in #vcs on Slack, and can cause issues for others working on project branches. This commit updates the playbook to instead create a fresh clone of the repo using stream clones on both hgssh and hgweb, speeding up the process and working around the vcsreplicator queue lag. As a result of this change, we also run the `repo-permissions` script on hgweb and earlier in the reset process on hgssh. I used this script to resolve bug 1672465 without running into the aforementioned problems. Differential Revision: https://phabricator.services.mozilla.com/D94523
ansible/hgmo-reset-twig.yml
--- a/ansible/hgmo-reset-twig.yml
+++ b/ansible/hgmo-reset-twig.yml
@@ -47,50 +47,46 @@
 
 - hosts: hgssh-prod-master:hgweb-prod:hgweb-mirrors
   gather_facts: no
   tasks:
     - name: Rename twig to twig-old
       command: mv /repo/hg/mozilla/projects/{{ twig }} /repo/hg/mozilla/projects/{{ twig }}-old
                creates=/repo/hg/mozilla/projects/{{ twig }}-old
       tags: rename
+    
+    - name: stream clone repository
+      command: /var/hg/venv_tools/bin/hg --config extensions.vcsreplicator=! --config extensions.pushlog=! --config extensions.replicateowner=! --config extensions.readonly=! clone -U --stream https://hg.mozilla.org/{{ hostvars['localhost'].source_repo }} /repo/hg/mozilla/projects/{{ twig }}
+
+- hosts: hgweb-prod:hgweb-mirrors
+  gather_facts: no
+  tasks:
+    - name: Set permissions
+      command: /var/hg/version-control-tools/scripts/repo-permissions /repo/hg/mozilla/projects/{{ twig }} hg hg wwr
 
 - hosts: hgssh-prod-master
   gather_facts: no
   tasks:
-    - name: Initialize new twig repo
-      become_user: hg
-      command: /var/hg/venv_tools/bin/hg init /repo/hg/mozilla/projects/{{ twig }}
-               creates=/repo/hg/mozilla/projects/{{ twig }}
-      tags: reset
-
-    - name: Pull new repo
-      become_user: hg
-      command: /var/hg/venv_tools/bin/hg --config extensions.vcsreplicator=! pull -r {{ revision | default("tip") }} /repo/hg/mozilla/{{ hostvars['localhost'].source_repo }}
-               chdir=/repo/hg/mozilla/projects/{{ twig }}
-      register: hg_pull
+    - name: Set permissions
+      command: /var/hg/version-control-tools/scripts/repo-permissions /repo/hg/mozilla/projects/{{ twig }} hg scm_level_2 wwr
       tags: reset
 
     - name: Create hgrc
       become_user: hg
       ini_file: dest=/repo/hg/mozilla/projects/{{ twig }}/.hg/hgrc
                 section={{ item.section }}
                 option={{ item.option }}
                 value={{ item.value }}
                 owner=hg
                 group=scm_level_2
       with_items:
         - { section: paths, option: default, value: "/repo/hg/mozilla/{{ hostvars['localhost'].source_repo }}" }
         - { section: hooks, option: pretxnchangegroup.a_treeclosure, value: "python:mozhghooks.treeclosure.hook" }
       tags: reset
 
-    - name: Set permissions
-      command: /var/hg/version-control-tools/scripts/repo-permissions /repo/hg/mozilla/projects/{{ twig }} hg scm_level_2 wwr
-      tags: reset
-
     - name: Check revision
       command: /var/hg/venv_tools/bin/hg --config extensions.vcsreplicator=! log -l 1
                chdir=/repo/hg/mozilla/projects/{{ twig }}
       register: hg_log
       tags: reset
 
     - name: Run replicatesync
       command: /var/hg/venv_tools/bin/hg replicatesync