vcsreplicator: add test for filters behaviour of "all-allowed" default
Also update the rule to be correct for on-prem hosts, and fix Ansible
configs to account for change in the variable.
--- a/ansible/roles/hg-web/defaults/main.yml
+++ b/ansible/roles/hg-web/defaults/main.yml
@@ -23,9 +23,9 @@ mirror_private_key: "undefined"
# Bloxtool connection settings (so we can retrieve Mozilla office IPs).
# Actual values will get pulled in from secrets.
bloxtool_user: bloxtool_user
bloxtool_password: bloxtool_password
bloxtool_url: https://bloxtool
# Flag used to turn on/off replication filters on a host, default off
-replication_filters: False
+replication_filters: null
--- a/ansible/roles/hg-web/templates/hgrc.j2
+++ b/ansible/roles/hg-web/templates/hgrc.j2
@@ -102,15 +102,15 @@ web.api.http-v2 = true
# obsolescence markers, even if not enabled for regular users.
obsolescenceexchangeusers = hg
[hgmo]
awsippath = /var/hg/aws-ip-ranges.json
gcpippath = /var/hg/gcp-ip-ranges.txt
pullclonebundlesmanifest = true
-{% if replication_filters == True %}
+{% if replication_filters == "mirror" %}
# Set the instance data path only on mirrors
instance-data-path = /var/hg/instance_data.json
{% endif %}
[mozilla]
firefoxreleasesdb = /var/hg/fxreleases/db.db
--- a/ansible/roles/hg-web/templates/on-premises-filters.j2
+++ b/ansible/roles/hg-web/templates/on-premises-filters.j2
@@ -1,4 +1,4 @@
[replicationrules]
exclude.pine = path:{moz}/projects/pine
-include.all = re:{moz}/*
+include.all = re:\{moz\}/.*
--- a/ansible/roles/hg-web/templates/vcsreplicator-pending.ini.j2
+++ b/ansible/roles/hg-web/templates/vcsreplicator-pending.ini.j2
@@ -10,11 +10,13 @@ topic = replicatedpushdatapending
# Every machine acting on replication events needs to receive its own
# copy of the message. So the group name must be unique.
group = {{ ansible_hostname }}
[path_rewrites]
{moz} = /repo/hg/mozilla
-{% if replication_filters == True %}
+{% if replication_filters == "mirror" %}
{% include 'mirror-filters.j2' %}
+{% elif replication_filters == "on-premises" %}
+{% include 'on-premises-filters.j2' %}
{% endif %}
--- a/pylib/vcsreplicator/tests/helpers.sh
+++ b/pylib/vcsreplicator/tests/helpers.sh
@@ -62,16 +62,30 @@ client_id = filteredconsumer-local
topic = pushdata
group = filteredconsumer
[replicationrules]
include.projects = re:\{moz\}/projects/.*
include.central = path:{moz}/mozilla-central
exclude.unified = path:{moz}/mozilla-unified
exclude.users = re:\{moz\}/users/.*
EOF
+
+cat >> filtered-consumer-default.ini << EOF
+[consumer]
+hosts = ${KAFKA_0_HOSTPORT}, ${KAFKA_1_HOSTPORT}, ${KAFKA_2_HOSTPORT}
+connect_timeout = 5
+poll_timeout = 0.2
+client_id = filteredconsumerdefault-local
+topic = pushdata
+group = filteredconsumerdefault
+[replicationrules]
+exclude.ash = path:{moz}/projects/ash
+include.all = re:\{moz\}/.*
+EOF
}
alias consumer='vcsreplicator-consumer $TESTTMP/vcsreplicator.ini'
alias papendingconsumer='vcsreplicator-consumer $TESTTMP/pushdataaggregator-pendingconsumer.ini'
alias paconsumer='vcsreplicator-consumer $TESTTMP/pushdataaggregator-consumer.ini'
alias pulseconsumer='vcsreplicator-consumer $TESTTMP/pulse-consumer.ini'
alias snsconsumer='vcsreplicator-consumer $TESTTMP/sns-consumer.ini'
alias filteredconsumer='vcsreplicator-consumer $TESTTMP/filtered-consumer.ini'
+alias filteredconsumerdefault='vcsreplicator-consumer $TESTTMP/filtered-consumer-default.ini'
--- a/pylib/vcsreplicator/tests/test-filter.t
+++ b/pylib/vcsreplicator/tests/test-filter.t
@@ -39,17 +39,17 @@ Dump state to confirm N replication mess
generaldelta: true
name: hg-repo-init-2
path: '{moz}/mozilla-central'
- _created: * (glob)
generaldelta: true
name: hg-repo-init-2
path: '{moz}/users/cosheehan_mozilla.com/vct'
-Use "onetime" consumer N times
+Use "onetime" consumer N times for mirror-like host
$ filteredconsumer --start-from 0 --onetime
vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 0
$ filteredconsumer --onetime
vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 1
$ filteredconsumer --onetime
vcsreplicator.consumer processing hg-repo-init-2 from partition 0 offset 2
vcsreplicator.consumer repo {moz}/mozilla-unified filtered by rule unified
@@ -62,11 +62,34 @@ Use "onetime" consumer N times
vcsreplicator.consumer created Mercurial repository: {moz}/projects/ash
$ filteredconsumer --onetime
vcsreplicator.consumer processing hg-repo-init-2 from partition 2 offset 0
vcsreplicator.consumer created Mercurial repository: {moz}/mozilla-central
$ filteredconsumer --onetime
vcsreplicator.consumer processing hg-repo-init-2 from partition 5 offset 0
vcsreplicator.consumer repo {moz}/users/cosheehan_mozilla.com/vct filtered by rule users
+Use "onetime" consumer N times for non-mirror-like host
+
+ $ filteredconsumerdefault --start-from 0 --onetime
+ vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 0
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 1
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing hg-repo-init-2 from partition 0 offset 2
+ vcsreplicator.consumer created Mercurial repository: {moz}/mozilla-unified
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 3
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing heartbeat-1 from partition 0 offset 4
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing hg-repo-init-2 from partition 0 offset 5
+ vcsreplicator.consumer repo {moz}/projects/ash filtered by rule ash
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing hg-repo-init-2 from partition 2 offset 0
+ vcsreplicator.consumer repository already exists: {moz}/mozilla-central
+ $ filteredconsumerdefault --onetime
+ vcsreplicator.consumer processing hg-repo-init-2 from partition 5 offset 0
+ vcsreplicator.consumer created Mercurial repository: {moz}/users/cosheehan_mozilla.com/vct
+
Clean
$ hgmo clean