vcsreplicator: add test for filters behaviour of "all-allowed" default
authorConnor Sheehan <sheehan@mozilla.com>
Mon, 01 Mar 2021 13:31:23 -0500
changeset 7685 e404c980045c5ad6242f436344970cb5a73058c9
parent 7684 7604a073b6cbe36d3100ad6a13483614e7a89d67
child 7686 65c9aa4f8f20384bc8633fdd354871a4f956a558
push id3845
push usercosheehan@mozilla.com
push dateMon, 01 Mar 2021 18:33:38 +0000
treeherderversion-control-tools@e404c980045c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
vcsreplicator: add test for filters behaviour of "all-allowed" default Also update the rule to be correct for on-prem hosts, and fix Ansible configs to account for change in the variable.
ansible/roles/hg-web/defaults/main.yml
ansible/roles/hg-web/templates/hgrc.j2
ansible/roles/hg-web/templates/on-premises-filters.j2
ansible/roles/hg-web/templates/vcsreplicator-pending.ini.j2
pylib/vcsreplicator/tests/helpers.sh
pylib/vcsreplicator/tests/test-filter.t
--- a/ansible/roles/hg-web/defaults/main.yml
+++ b/ansible/roles/hg-web/defaults/main.yml
@@ -23,9 +23,9 @@ mirror_private_key: "undefined"
 
 # Bloxtool connection settings (so we can retrieve Mozilla office IPs).
 # Actual values will get pulled in from secrets.
 bloxtool_user: bloxtool_user
 bloxtool_password: bloxtool_password
 bloxtool_url: https://bloxtool
 
 # Flag used to turn on/off replication filters on a host, default off
-replication_filters: False
+replication_filters: null
--- a/ansible/roles/hg-web/templates/hgrc.j2
+++ b/ansible/roles/hg-web/templates/hgrc.j2
@@ -102,15 +102,15 @@ web.api.http-v2 = true
 # obsolescence markers, even if not enabled for regular users.
 obsolescenceexchangeusers = hg
 
 [hgmo]
 awsippath = /var/hg/aws-ip-ranges.json
 gcpippath = /var/hg/gcp-ip-ranges.txt
 pullclonebundlesmanifest = true
 
-{% if replication_filters == True %}
+{% if replication_filters == "mirror" %}
 # Set the instance data path only on mirrors
 instance-data-path = /var/hg/instance_data.json
 {% endif %}
 
 [mozilla]
 firefoxreleasesdb = /var/hg/fxreleases/db.db
--- a/ansible/roles/hg-web/templates/on-premises-filters.j2
+++ b/ansible/roles/hg-web/templates/on-premises-filters.j2
@@ -1,4 +1,4 @@
 [replicationrules]
 exclude.pine = path:{moz}/projects/pine
-include.all = re:{moz}/*
+include.all = re:\{moz\}/.*
 
--- a/ansible/roles/hg-web/templates/vcsreplicator-pending.ini.j2
+++ b/ansible/roles/hg-web/templates/vcsreplicator-pending.ini.j2
@@ -10,11 +10,13 @@ topic = replicatedpushdatapending
 
 # Every machine acting on replication events needs to receive its own
 # copy of the message. So the group name must be unique.
 group = {{ ansible_hostname }}
 
 [path_rewrites]
 {moz} = /repo/hg/mozilla
 
-{% if replication_filters == True %}
+{% if replication_filters == "mirror" %}
 {% include 'mirror-filters.j2' %}
+{% elif replication_filters == "on-premises" %}
+{% include 'on-premises-filters.j2' %}
 {% endif %}
--- a/pylib/vcsreplicator/tests/helpers.sh
+++ b/pylib/vcsreplicator/tests/helpers.sh
@@ -62,16 +62,30 @@ client_id = filteredconsumer-local
 topic = pushdata
 group = filteredconsumer
 [replicationrules]
 include.projects = re:\{moz\}/projects/.*
 include.central = path:{moz}/mozilla-central
 exclude.unified = path:{moz}/mozilla-unified
 exclude.users = re:\{moz\}/users/.*
 EOF
+
+cat >> filtered-consumer-default.ini << EOF
+[consumer]
+hosts = ${KAFKA_0_HOSTPORT}, ${KAFKA_1_HOSTPORT}, ${KAFKA_2_HOSTPORT}
+connect_timeout = 5
+poll_timeout = 0.2
+client_id = filteredconsumerdefault-local
+topic = pushdata
+group = filteredconsumerdefault
+[replicationrules]
+exclude.ash = path:{moz}/projects/ash
+include.all = re:\{moz\}/.*
+EOF
 }
 
 alias consumer='vcsreplicator-consumer $TESTTMP/vcsreplicator.ini'
 alias papendingconsumer='vcsreplicator-consumer $TESTTMP/pushdataaggregator-pendingconsumer.ini'
 alias paconsumer='vcsreplicator-consumer $TESTTMP/pushdataaggregator-consumer.ini'
 alias pulseconsumer='vcsreplicator-consumer $TESTTMP/pulse-consumer.ini'
 alias snsconsumer='vcsreplicator-consumer $TESTTMP/sns-consumer.ini'
 alias filteredconsumer='vcsreplicator-consumer $TESTTMP/filtered-consumer.ini'
+alias filteredconsumerdefault='vcsreplicator-consumer $TESTTMP/filtered-consumer-default.ini'
--- a/pylib/vcsreplicator/tests/test-filter.t
+++ b/pylib/vcsreplicator/tests/test-filter.t
@@ -39,17 +39,17 @@ Dump state to confirm N replication mess
     generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
   - _created: * (glob)
     generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/users/cosheehan_mozilla.com/vct'
 
-Use "onetime" consumer N times
+Use "onetime" consumer N times for mirror-like host
 
   $ filteredconsumer --start-from 0 --onetime
   vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 0
   $ filteredconsumer --onetime
   vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 1
   $ filteredconsumer --onetime
   vcsreplicator.consumer processing hg-repo-init-2 from partition 0 offset 2
   vcsreplicator.consumer repo {moz}/mozilla-unified filtered by rule unified
@@ -62,11 +62,34 @@ Use "onetime" consumer N times
   vcsreplicator.consumer created Mercurial repository: {moz}/projects/ash
   $ filteredconsumer --onetime
   vcsreplicator.consumer processing hg-repo-init-2 from partition 2 offset 0
   vcsreplicator.consumer created Mercurial repository: {moz}/mozilla-central
   $ filteredconsumer --onetime
   vcsreplicator.consumer processing hg-repo-init-2 from partition 5 offset 0
   vcsreplicator.consumer repo {moz}/users/cosheehan_mozilla.com/vct filtered by rule users
 
+Use "onetime" consumer N times for non-mirror-like host
+
+  $ filteredconsumerdefault --start-from 0 --onetime
+  vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 0
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 1
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing hg-repo-init-2 from partition 0 offset 2
+  vcsreplicator.consumer created Mercurial repository: {moz}/mozilla-unified
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 3
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 4
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing hg-repo-init-2 from partition 0 offset 5
+  vcsreplicator.consumer repo {moz}/projects/ash filtered by rule ash
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing hg-repo-init-2 from partition 2 offset 0
+  vcsreplicator.consumer repository already exists: {moz}/mozilla-central
+  $ filteredconsumerdefault --onetime
+  vcsreplicator.consumer processing hg-repo-init-2 from partition 5 offset 0
+  vcsreplicator.consumer created Mercurial repository: {moz}/users/cosheehan_mozilla.com/vct
+
 Clean
 
   $ hgmo clean