Bug 1636158 - Update the addons-mlbf.bin attachment via periodic_file_updates.sh r=sfraser
authorRob Wu <rob@robwu.nl>
Fri, 22 May 2020 01:37:37 +0000
changeset 531642 d3770d94e7c0dc8a5b06b526a01cde67de83104b
parent 531641 d07f8d9d1a01068fdad27c09966326b7109574a5
child 531643 049bdaa3e410558ddcb1a71bbc26f3a16b13d0ed
push id37441
push userapavel@mozilla.com
push dateFri, 22 May 2020 21:38:53 +0000
treeherdermozilla-central@d6abd35b54ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfraser
bugs1636158
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1636158 - Update the addons-mlbf.bin attachment via periodic_file_updates.sh r=sfraser Tested by running the following code and verifying that the script runs successfully and generates a reasonable diff with the latest metadata and matching attachment content: ``` $ cd taskcluster/docker/periodic-updates/ $ tar -czf- -s "#$(git rev-parse --show-cdup)#topsrcdir/#" . $(awk -v d="$(git rev-parse --show-cdup)" '/^# %include/{print d$3}' Dockerfile) | docker build - -t hsts-local $ docker run -e DO_REMOTE_SETTINGS=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local ``` After that, I opened a shell in the container, built Firefox and ran the following test, to confirm that the updated records are used correctly: ``` mach test toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js ``` Differential Revision: https://phabricator.services.mozilla.com/D76158
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -319,27 +319,91 @@ function compare_remote_settings_files {
       continue
     fi
 
     # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
     remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
     local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
     mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
     ${WGET} -qO "$local_location_output" "$remote_records_url"
+
+    # 5. Download attachments if needed.
+    if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then
+      # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.jsm.
+      # The server should return one "bloomfilter-base" record, but in case it returns multiple,
+      # return the most recent one. The server may send multiple entries if we ever decide to use
+      # the "filter_expression" feature of Remote Settings to send different records to specific
+      # channels. In that case this code should be updated to recognize the filter expression,
+      # but until we do, simply select the most recent record - can't go wrong with that.
+      # Note that "attachment_type" and "generation_time" are specific to addons-bloomfilters.
+      update_remote_settings_attachment "${bucket}" "${collection}" addons-mlbf.bin \
+        'map(select(.attachment_type == "bloomfilter-base")) | sort_by(.generation_time) | last'
+    fi
+    # Here is an example to download an attachment with record identifier "ID":
+    # update_remote_settings_attachment "${bucket}" "${collection}" ID '.[] | select(.id == "ID")'
+    # NOTE: The downloaded data is not validated. xpcshell should be used for that.
   done
 
   echo "INFO: diffing old/new remote settings dumps..."
   ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
   if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
   then
     return 0
   fi
   return 1
 }
 
+# Helper for compare_remote_settings_files to download attachments from remote settings.
+# The format and location is documented at:
+# https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#packaging-attachments
+function update_remote_settings_attachment() {
+  local bucket=$1
+  local collection=$2
+  local attachment_id=$3
+  # $4 is a jq filter on the arrays that should return one record with the attachment
+  local jq_attachment_selector=".data | map(select(.attachment)) | $4"
+
+  # These paths match _readAttachmentDump in services/settings/Attachments.jsm.
+  local path_to_attachment="${bucket}/${collection}/${attachment_id}"
+  local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json"
+  local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}"
+  local new_meta="$REMOTE_SETTINGS_OUTPUT/${path_to_meta}"
+
+  # Those files should have been created by compare_remote_settings_files before the function call.
+  local local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
+  local local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
+
+  # Compute the metadata based on already-downloaded records.
+  mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}/${collection}"
+  ${JQ} -cj <"$local_location_input" "${jq_attachment_selector}" > "${old_meta}"
+  mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}"
+  ${JQ} -cj <"$local_location_output" "${jq_attachment_selector}" > "${new_meta}"
+
+  if cmp --silent "${old_meta}" "${new_meta}" ; then
+    # Metadata not changed, don't bother downloading the attachments themselves.
+    return
+  fi
+  # Metadata changed. Download attachments.
+
+  echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}"
+
+  # Overwrited old_meta with the actual file from the repo. The content should be equivalent,
+  # but can have minor differences (e.g. different line endings) if the checked in file was not
+  # generated by this script (e.g. manually checked in).
+  ${WGET} -qO "${old_meta}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_meta}"
+
+  ${WGET} -qO "${REMOTE_SETTINGS_INPUT}/${path_to_attachment}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_attachment}"
+
+  if [ -z "${ATTACHMENT_BASE_URL}" ] ; then
+    ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url)
+  fi
+  attachment_path_from_meta=$(${JQ} -r < "${new_meta}" .attachment.location)
+  ${WGET} -qO "${REMOTE_SETTINGS_OUTPUT}/${path_to_attachment}" "${ATTACHMENT_BASE_URL}${attachment_path_from_meta}"
+}
+
 # Clones an hg repo
 function clone_repo {
   cd "${BASEDIR}"
   if [ ! -d "${REPODIR}" ]; then
     ${HG} robustcheckout --sharebase /tmp/hg-store -b default "${HGREPO}" "${REPODIR}"
   fi
 
   ${HG} -R ${REPODIR} pull
--- a/toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js
@@ -4,16 +4,17 @@
 "use strict";
 
 /**
  * @fileOverview Verifies that the MLBF dump of the addons blocklist is
  * correctly registered.
  */
 
 Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
+Services.prefs.setBoolPref("extensions.blocklist.useMLBF.stashes", true);
 
 const { ExtensionBlocklist: ExtensionBlocklistMLBF } = Blocklist;
 
 // A known blocked version from bug 1626602.
 const blockedAddon = {
   id: "{6f62927a-e380-401a-8c9e-c485b7d87f0d}",
   version: "9.2.0",
   // The following date is the date of the first checked in MLBF. Any MLBF