Bug 1618188 - stop updating the mozilla-central copy of blocklist.xml, r=RyanVM
authorGijs Kruitbosch <gijskruitbosch@gmail.com>
Mon, 09 Mar 2020 11:58:32 +0000
changeset 517522 cbe2fe9d1c540370dcb6dcfb4db2a1598c44aa86
parent 517521 337b6019e3d4e4fc738eb8c7cdc679a9d1f5a18c
child 517523 06ca3c111fc7b2fd2c2c8137bdac13f8d52bfb98
push id37198
push useropoprus@mozilla.com
push dateMon, 09 Mar 2020 21:52:54 +0000
treeherdermozilla-central@268543e53e1b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersRyanVM
bugs1618188
milestone75.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1618188 - stop updating the mozilla-central copy of blocklist.xml, r=RyanVM Differential Revision: https://phabricator.services.mozilla.com/D64932
taskcluster/ci/repo-update/kind.yml
taskcluster/docker/periodic-updates/README.md
taskcluster/docker/periodic-updates/runme.sh
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
--- a/taskcluster/ci/repo-update/kind.yml
+++ b/taskcluster/ci/repo-update/kind.yml
@@ -15,20 +15,16 @@ job-defaults:
             DO_HSTS:
                 by-project:
                     mozilla-(central|beta|esr.*): "1"
                     default: ""
             DO_HPKP:
                 by-project:
                     mozilla-(central|beta|esr.*): "1"
                     default: ""
-            DO_BLOCKLIST:
-                by-project:
-                    mozilla-(central|beta|release|esr.*): "1"
-                    default: ""
             DO_REMOTE_SETTINGS:
                 by-project:
                     mozilla-(central|beta|release|esr.*): "1"
                     default: ""
             DO_SUFFIX_LIST:
                 by-project:
                     mozilla-(central|beta|esr.*): "1"
                     default: ""
@@ -36,17 +32,17 @@ job-defaults:
                 by-project:
                     mozilla-central: "1"
                     default: ""
 
 
 jobs:
     hsts-hpkp-blocklist:
         name: periodic_file_update
-        description: HSTS, HPKP, Blocklist and remote settings update
+        description: HSTS, HPKP and remote settings update
         run-on-projects: []  # Only run via cron
         treeherder:
             kind: build
             platform: linux64/opt
             symbol: pfu
             tier: 1
         worker-type: b-linux
         worker:
@@ -62,16 +58,13 @@ jobs:
             taskcluster-proxy: true
             artifacts:
                 - name: 'public/build/nsSTSPreloadList.diff'
                   path: '/home/worker/artifacts/nsSTSPreloadList.diff'
                   type: file
                 - name: 'public/build/StaticHPKPins.h.diff'
                   path: '/home/worker/artifacts/StaticHPKPins.h.diff'
                   type: file
-                - name: 'public/build/blocklist.diff'
-                  path: '/home/worker/artifacts/blocklist.diff'
-                  type: file
                 - name: 'public/build/remote-settings.diff'
                   path: '/home/worker/artifacts/remote-settings.diff'
                   type: file
         scopes:
             - secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token
--- a/taskcluster/docker/periodic-updates/README.md
+++ b/taskcluster/docker/periodic-updates/README.md
@@ -1,29 +1,29 @@
 
 ==Periodic File Updates==
 
-This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklist.xml, and
+This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklisting, and
 will produce a diff for each necessary to update the in-tree files.
 
 If given a conduit API token, it will also use the arcanist client to submit the commits for review.
 
 
 ==Quick Start==
 
 ```sh
 docker build -t hsts-local --no-cache --rm .
 
-docker run -e DO_HSTS=1 -e DO_HPKP=1 -e DO_BLOCKLIST=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local
+docker run -e DO_HSTS=1 -e DO_HPKP=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local
 ```
 
 HSTS checks will only be run if the `DO_HSTS` environment variable is set.
-Likewise for `DO_HPKP` and the HPKP checks, and `DO_BLOCKLIST` and the
-blocklist checks. Environment variables are used rather than command line
-arguments to make constructing taskcluster tasks easier.
+Likewise for `DO_HPKP` and the HPKP checks. Environment variables are used
+rather than command line arguments to make constructing taskcluster tasks
+easier.
 
 To prevent a full build when landing with Phabricator, set the `DONTBUILD`
 environment variable.
 
 ==Background==
 
 These scripts have been moved from
 `https://hg.mozilla.org/build/tools/scripts/periodic_file_updates/` and
@@ -70,17 +70,16 @@ payload:
       type: file
     public/build/blocklist.diff:
       path: /home/worker/artifacts/blocklist.diff
       expires: '2019-02-07T13:57:35.448Z'
       type: file
   env:
     DO_HSTS: 1
     DO_HPKP: 1
-    DO_BLOCKLIST: 1
     PRODUCT: firefox
     BRANCH: mozilla-central
     USE_MOZILLA_CENTRAL: 1
     REVIEWERS: catlee
 metadata:
   name: Periodic updates testing
   description: Produce diffs for HSTS and HPKP in-tree files.
   owner: sfraser@mozilla.com
--- a/taskcluster/docker/periodic-updates/runme.sh
+++ b/taskcluster/docker/periodic-updates/runme.sh
@@ -1,14 +1,14 @@
 #!/bin/bash
 
 set -xe
 
 # Things to be set by task definition.
-# --pinset --hsts --hpkp --blocklist
+# --pinset --hsts --hpkp
 # -b branch
 # --use-mozilla-central
 # -p firefox
 # Artifact directory
 # Artifact names.
 
 
 test "${BRANCH}"
@@ -27,21 +27,16 @@ then
   PARAMS="${PARAMS} --hsts"
 fi
 
 if [ -n "${DO_HPKP}" ]
 then
   PARAMS="${PARAMS} --hpkp"
 fi
 
-if [ -n "${DO_BLOCKLIST}" ]
-then
-  PARAMS="${PARAMS} --blocklist"
-fi
-
 if [ -n "${DO_REMOTE_SETTINGS}" ]
 then
   PARAMS="${PARAMS} --remote-settings"
 fi
 
 if [ -n "${DO_SUFFIX_LIST}" ]
 then
   PARAMS="${PARAMS} --suffix-list"
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -9,34 +9,32 @@ Usage: $(basename "$0") -h # Displays th
 Usage: $(basename "$0") -x # lists exit codes
 Usage: $(basename "$0") [-p product]
            [-r existing_repo_dir]
            # Use mozilla-central builds to check HSTS & HPKP
            [--use-mozilla-central]
            # Use archive.m.o instead of the taskcluster index to get xpcshell
            [--use-ftp-builds]
            # One (or more) of the following actions must be specified.
-           --hsts | --hpkp | --blocklist
+           --hsts | --hpkp | --remote-settings | --suffix-list
            -b branch
 
 EOF
 }
 
 PRODUCT="firefox"
 BRANCH=""
 PLATFORM_EXT="tar.bz2"
 UNPACK_CMD="tar jxf"
 CLOSED_TREE=false
 DONTBUILD=false
 APPROVAL=false
 COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>'
 REPODIR=''
 APP_DIR=''
-APP_ID=''
-APP_NAME=''
 HGHOST="hg.mozilla.org"
 STAGEHOST="archive.mozilla.org"
 WGET="wget -nv"
 UNTAR="tar -zxf"
 DIFF="$(command -v diff) -u"
 BASEDIR="${HOME}"
 
 SCRIPTDIR="$(realpath "$(dirname "$0")")"
@@ -60,23 +58,16 @@ DO_HPKP=false
 HPKP_PRELOAD_SCRIPT="${SCRIPTDIR}/genHPKPStaticPins.js"
 HPKP_PRELOAD_ERRORS="StaticHPKPins.errors"
 HPKP_PRELOAD_JSON="${DATADIR}/PreloadedHPKPins.json"
 HPKP_PRELOAD_INC="StaticHPKPins.h"
 HPKP_PRELOAD_INPUT="${DATADIR}/${HPKP_PRELOAD_INC}"
 HPKP_PRELOAD_OUTPUT="${DATADIR}/${HPKP_PRELOAD_INC}.out"
 HPKP_UPDATED=false
 
-DO_BLOCKLIST=false
-BLOCKLIST_URL_AMO=''
-BLOCKLIST_URL_HG=''
-BLOCKLIST_LOCAL_AMO="blocklist_amo.xml"
-BLOCKLIST_LOCAL_HG="blocklist_hg.xml"
-BLOCKLIST_UPDATED=false
-
 DO_REMOTE_SETTINGS=false
 REMOTE_SETTINGS_SERVER=''
 REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
 REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
 REMOTE_SETTINGS_DIR="/services/settings/dumps"
 REMOTE_SETTINGS_UPDATED=false
 
 DO_SUFFIX_LIST=false
@@ -85,17 +76,16 @@ GITHUB_SUFFIX_LOCAL="public_suffix_list.
 HG_SUFFIX_LOCAL="effective_tld_names.dat"
 HG_SUFFIX_PATH="/netwerk/dns/${HG_SUFFIX_LOCAL}"
 SUFFIX_LIST_UPDATED=false
 
 ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
 # Defaults
 HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
 HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
-BLOCKLIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${BLOCKLIST_DIFF_ARTIFACT:-"blocklist.diff"}"
 REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
 SUFFIX_LIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${SUFFIX_LIST_DIFF_ARTIFACT:-"effective_tld_names.diff"}"
 
 # duplicate the functionality of taskcluster-lib-urls, but in bash..
 queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1"
 index_base="$TASKCLUSTER_ROOT_URL/api/index/v1"
 
 # Get the current in-tree version for a code branch.
@@ -293,56 +283,25 @@ function compare_suffix_lists {
   echo "INFO: ${WGET} -O ${GITHUB_SUFFIX_LOCAL} ${GITHUB_SUFFIX_URL}"
   rm -f "${GITHUB_SUFFIX_LOCAL}"
   ${WGET} -O "${GITHUB_SUFFIX_LOCAL}" "${GITHUB_SUFFIX_URL}"
 
   echo "INFO: ${WGET} -O ${HG_SUFFIX_LOCAL} ${HG_SUFFIX_URL}"
   rm -f "${HG_SUFFIX_LOCAL}"
   ${WGET} -O "${HG_SUFFIX_LOCAL}" "${HG_SUFFIX_URL}"
 
-  echo "INFO: diffing in-tree blocklist against the blocklist from AMO..."
+  echo "INFO: diffing in-tree suffix list against the suffix list from AMO..."
   ${DIFF} ${GITHUB_SUFFIX_LOCAL} ${HG_SUFFIX_LOCAL} | tee "${SUFFIX_LIST_DIFF_ARTIFACT}"
   if [ -s "${SUFFIX_LIST_DIFF_ARTIFACT}" ]
   then
     return 0
   fi
   return 1
 }
 
-# Downloads the current in-tree blocklist file.
-# Downloads the current blocklist file from AMO.
-# Compares the AMO blocklist with the in-tree blocklist to determine whether we need to update.
-function compare_blocklist_files {
-  BLOCKLIST_URL_AMO="https://blocklist.addons.mozilla.org/blocklist/3/${APP_ID}/${VERSION}/${APP_NAME}/20090105024647/blocklist-sync/en-US/nightly/blocklist-sync/default/default/"
-  BLOCKLIST_URL_HG="${HGREPO}/raw-file/default/${APP_DIR}/app/blocklist.xml"
-
-  cd "${BASEDIR}"
-  rm -f ${BLOCKLIST_LOCAL_AMO}
-  echo "INFO: ${WGET} -O ${BLOCKLIST_LOCAL_AMO} ${BLOCKLIST_URL_AMO}"
-  ${WGET} -O "${BLOCKLIST_LOCAL_AMO}" "${BLOCKLIST_URL_AMO}"
-
-  rm -f ${BLOCKLIST_LOCAL_HG}
-  echo "INFO: ${WGET} -O ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_URL_HG}"
-  ${WGET} -O "${BLOCKLIST_LOCAL_HG}" "${BLOCKLIST_URL_HG}"
-
-  # The downloaded files should be non-empty and have a valid xml header
-  # if they were retrieved properly, and some random HTML garbage if not.
-  # set -x catches these
-  is_valid_xml ${BLOCKLIST_LOCAL_AMO}
-  is_valid_xml ${BLOCKLIST_LOCAL_HG}
-
-  echo "INFO: diffing in-tree blocklist against the blocklist from AMO..."
-  ${DIFF} ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_LOCAL_AMO} | tee "${BLOCKLIST_DIFF_ARTIFACT}"
-  if [ -s "${BLOCKLIST_DIFF_ARTIFACT}" ]
-  then
-    return 0
-  fi
-  return 1
-}
-
 function compare_remote_settings_files {
   REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
 
   # 1. List remote settings collections from server.
   echo "INFO: fetch remote settings list from server"
   ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
     ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
     # 2. For each entry ${bucket, collection}
@@ -393,21 +352,16 @@ function stage_hsts_files {
   cp -f "${BASEDIR}/${PRODUCT}/$(basename "${HSTS_PRELOAD_INC}")" "${REPODIR}/security/manager/ssl/"
 }
 
 function stage_hpkp_files {
   cd "${BASEDIR}"
   cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}"
 }
 
-function stage_blocklist_files {
-  cd "${BASEDIR}"
-  cp -f ${BLOCKLIST_LOCAL_AMO} ${REPODIR}/${APP_DIR}/app/blocklist.xml
-}
-
 function stage_remote_settings_files {
   cd "${BASEDIR}"
   cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}"
 }
 
 function stage_tld_suffix_files {
   cd "${BASEDIR}"
   cp -a "${GITHUB_SUFFIX_LOCAL}" "${REPODIR}/${HG_SUFFIX_PATH}"
@@ -453,17 +407,16 @@ while [ $# -gt 0 ]; do
     -b) BRANCH="$2"; shift ;;
     -n) DRY_RUN=true ;;
     -c) CLOSED_TREE=true ;;
     -d) DONTBUILD=true ;;
     -a) APPROVAL=true ;;
     --pinset) DO_PRELOAD_PINSET=true ;;
     --hsts) DO_HSTS=true ;;
     --hpkp) DO_HPKP=true ;;
-    --blocklist) DO_BLOCKLIST=true ;;
     --remote-settings) DO_REMOTE_SETTINGS=true ;;
     --suffix-list) DO_SUFFIX_LIST=true ;;
     -r) REPODIR="$2"; shift ;;
     --use-mozilla-central) USE_MC=true ;;
     --use-ftp-builds) USE_TC=false ;;
     -*) usage
       exit 11 ;;
     *)  break ;; # terminate while loop
@@ -474,35 +427,31 @@ done
 # Must supply a code branch to work with.
 if [ "${BRANCH}" == "" ]; then
   echo "Error: You must specify a branch with -b branchname." >&2
   usage
   exit 12
 fi
 
 # Must choose at least one update action.
-if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ]
+if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ]
 then
-  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist, --remote-settings" >&2
+  echo "Error: you must specify at least one action from: --hsts, --hpkp, --remote-settings, or --suffix-list" >&2
   usage
   exit 13
 fi
 
 # per-product constants
 case "${PRODUCT}" in
   thunderbird)
     APP_DIR="mail"
-    APP_ID="%7B3550f703-e582-4d05-9a08-453d09bdfdc6%7D"
-    APP_NAME="Thunderbird"
     COMMIT_AUTHOR="tbirdbld <tbirdbld@thunderbird.net>"
     ;;
   firefox)
     APP_DIR="browser"
-    APP_ID="%7Bec8030f7-c20a-464f-9b0e-13a3a9e97384%7D"
-    APP_NAME="Firefox"
     ;;
   *)
     echo "Error: Invalid product specified"
     usage
     exit 14
     ;;
 esac
 
@@ -565,37 +514,31 @@ if [ "${DO_HSTS}" == "true" ]; then
   fi
 fi
 if [ "${DO_HPKP}" == "true" ]; then
   if compare_hpkp_files
   then
     HPKP_UPDATED=true
   fi
 fi
-if [ "${DO_BLOCKLIST}" == "true" ]; then
-  if compare_blocklist_files
-  then
-    BLOCKLIST_UPDATED=true
-  fi
-fi
 if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
   if compare_remote_settings_files
   then
     REMOTE_SETTINGS_UPDATED=true
   fi
 fi
 if [ "${DO_SUFFIX_LIST}" == "true" ]; then
   if compare_suffix_lists
   then
     SUFFIX_LIST_UPDATED=true
   fi
 fi
 
 
-if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ]; then
+if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ]; then
   echo "INFO: no updates required. Exiting."
   exit 0
 else
   if [ "${DRY_RUN}" == "true" ]; then
     echo "INFO: Updates are available, not updating hg in dry-run mode."
     exit 2
   fi
 fi
@@ -610,22 +553,16 @@ then
 fi
 
 if [ "${HPKP_UPDATED}" == "true" ]
 then
   stage_hpkp_files
   COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP"
 fi
 
-if [ "${BLOCKLIST_UPDATED}" == "true" ]
-then
-  stage_blocklist_files
-  COMMIT_MESSAGE="${COMMIT_MESSAGE} blocklist"
-fi
-
 if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
 then
   stage_remote_settings_files
   COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings"
 fi
 
 if [ "${SUFFIX_LIST_UPDATED}" == "true" ]
 then