Bug 1467456 - Use correct hg repo in repo-update. r=lguo, a=release
authorSimon Fraser <sfraser@mozilla.com>
Thu, 07 Jun 2018 17:55:50 +0000
changeset 473665 7679eddb8fdbf61701a1d950e34f19564726edf1
parent 473664 0119d67ab929d6575b03c67a90fd7a5ffae9d7f9
child 473666 a2590a16f2d8ad4a792b358cca61bbb397cbbc94
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerslguo, release
bugs1467456
milestone61.0
Bug 1467456 - Use correct hg repo in repo-update. r=lguo, a=release 1. Updated hgrepo to work with mozilla-beta, mozilla-esr60 and project branches (just in case) 2. Presquashed commits, so we only submit one. 3. Replaced 'which' with 'command -v' to avoid future shellcheck issues. Differential Revision: https://phabricator.services.mozilla.com/D1582
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -31,31 +31,30 @@ HG_SSH_USER='ffxbld'
 REPODIR=''
 APP_DIR=''
 APP_ID=''
 APP_NAME=''
 HGHOST="hg.mozilla.org"
 STAGEHOST="archive.mozilla.org"
 WGET="wget -nv"
 UNZIP="unzip -q"
-DIFF="$(which diff) -u"
+DIFF="$(command -v diff) -u"
 BASEDIR="${HOME}"
 TOOLSDIR="${HOME}/tools"
-HGTOOL="${TOOLSDIR}/buildfarm/utils/hgtool.py"
 
 SCRIPTDIR="$(realpath "$(dirname "$0")")"
-HG="$(which hg)"
+HG="$(command -v hg)"
 DATADIR="${BASEDIR}/data"
 mkdir -p "${DATADIR}"
 
 VERSION=''
 MCVERSION=''
 USE_MC=false
 USE_TC=true
-JQ="$(which jq)"
+JQ="$(command -v jq)"
 
 DO_HSTS=false
 HSTS_PRELOAD_SCRIPT="${SCRIPTDIR}/getHSTSPreloadList.js"
 HSTS_PRELOAD_ERRORS="nsSTSPreloadList.errors"
 HSTS_PRELOAD_INC="${DATADIR}/nsSTSPreloadList.inc"
 HSTS_UPDATED=false
 
 DO_HPKP=false
@@ -69,21 +68,29 @@ HPKP_UPDATED=false
 
 DO_BLOCKLIST=false
 BLOCKLIST_URL_AMO=''
 BLOCKLIST_URL_HG=''
 BLOCKLIST_LOCAL_AMO="blocklist_amo.xml"
 BLOCKLIST_LOCAL_HG="blocklist_hg.xml"
 BLOCKLIST_UPDATED=false
 
-ARTIFACTS_DIR="${ARTIFACTS_DIR:-'.'}"
+DO_REMOTE_SETTINGS=false
+REMOTE_SETTINGS_SERVER=''
+REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
+REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
+REMOTE_SETTINGS_DIR="/services/settings/dumps"
+REMOTE_SETTINGS_UPDATED=false
+
+ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
 # Defaults
 HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
 HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
 BLOCKLIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${BLOCKLIST_DIFF_ARTIFACT:-"blocklist.diff"}"
+REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
 
 
 # Get the current in-tree version for a code branch.
 function get_version {
   VERSION_REPO=$1
   VERSION_FILE='version.txt'
 
   # TODO bypass temporary file
@@ -254,17 +261,17 @@ function compare_hpkp_files {
   then
     return 0
   fi
   return 1
 }
 
 function is_valid_xml {
   xmlfile=$1
-  XMLLINT=$(which xmllint 2>/dev/null | head -n1)
+  XMLLINT=$(command -v xmllint 2>/dev/null | head -n1)
 
   if [ ! -x "${XMLLINT}" ]; then
     echo "ERROR: xmllint not found in PATH"
     exit 60
   fi
   ${XMLLINT} --nonet --noout "${xmlfile}"
 }
 
@@ -294,126 +301,108 @@ function compare_blocklist_files {
   ${DIFF} ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_LOCAL_AMO} | tee "${BLOCKLIST_DIFF_ARTIFACT}"
   if [ -s "${BLOCKLIST_DIFF_ARTIFACT}" ]
   then
     return 0
   fi
   return 1
 }
 
-function clone_build_tools {
-    rm -fr "${TOOLSDIR}"
-    CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
-    ${CLONE_CMD}
+function compare_remote_settings_files {
+  REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
+
+  # 1. List remote settings collections from server.
+  echo "INFO: fetch remote settings list from server"
+  ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
+    ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
+    # 2. For each entry ${bucket, collection}
+  while IFS="/" read -r bucket collection; do
+
+    # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
+    hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
+    local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
+    mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
+    ${WGET} -qO "$local_location_input" "$hg_dump_url"
+    if [ $? -eq 8 ]; then
+      # We don't keep any dump for this collection, skip it.
+      # Try to clean up in case no collection in this bucket has dump.
+      rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
+      continue
+    fi
+
+    # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
+    remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
+    local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
+    mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
+    ${WGET} -qO "$local_location_output" "$remote_records_url"
+  done
+
+  echo "INFO: diffing old/new remote settings dumps..."
+  ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
+  if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
+  then
+    return 0
+  fi
+  return 1
 }
 
-# Clones an hg repo, using hgtool preferentially.
+function clone_build_tools {
+  rm -fr "${TOOLSDIR}"
+  CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
+  ${CLONE_CMD}
+}
+
+# Clones an hg repo
 function clone_repo {
   cd "${BASEDIR}"
   if [ ! -d "${REPODIR}" ]; then
-    CLONE_CMD=""
-    if [ -f "${HGTOOL}" ]; then
-      # Need to pass the default branch here to avoid pollution from buildprops.json
-      # when hgtool.py is run in production.
-      CLONE_CMD="${HGTOOL} --branch default"
-    else
-      echo "INFO: hgtool.py not found. Falling back to vanilla hg."
-      CLONE_CMD="${HG} clone"
-    fi
-    CLONE_CMD="${CLONE_CMD} ${HGREPO} ${REPODIR}"
+    CLONE_CMD="${HG} clone ${HGREPO} ${REPODIR}"
     ${CLONE_CMD}
   fi
 
   ${HG} -R ${REPODIR} pull
   ${HG} -R ${REPODIR} update -C default
 }
 
 # Copies new HSTS files in place, and commits them.
-function commit_hsts_files {
+function stage_hsts_files {
   cd "${BASEDIR}"
-
   cp -f "${BASEDIR}/${PRODUCT}/$(basename "${HSTS_PRELOAD_INC}")" "${REPODIR}/security/manager/ssl/"
-
-  COMMIT_MESSAGE="No bug, Automated HSTS preload list update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=hsts-update"
-  fi
-  echo "INFO: committing HSTS changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
-# Copies new HPKP files in place, and commits them.
-function commit_hpkp_files {
+function stage_hpkp_files {
   cd "${BASEDIR}"
-
   cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}"
-
-  COMMIT_MESSAGE="No bug, Automated HPKP preload list update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=hpkp-update"
-  fi
-  echo "INFO: committing HPKP changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
-# Copies new blocklist file in place, and commits it.
-function commit_blocklist_files {
+function stage_blocklist_files {
   cd "${BASEDIR}"
   cp -f ${BLOCKLIST_LOCAL_AMO} ${REPODIR}/${APP_DIR}/app/blocklist.xml
-  COMMIT_MESSAGE="No bug, Automated blocklist update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=blocklist-update"
-  fi
-  echo "INFO: committing blocklist changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
+}
+
+function stage_remote_settings_files {
+  cd "${BASEDIR}"
+  cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}"
 }
 
 # Push all pending commits to Phabricator
 function push_repo {
   cd "${REPODIR}"
   if [ ! -r "${HOME}/.arcrc" ]
   then
     return 1
   fi
-  if ! ARC=$(which arc)
+  if ! ARC=$(command -v arc)
   then
     return 1
   fi
   if [ -z "${REVIEWERS}" ]
   then
     return 1
   fi
-
   # Clean up older review requests
   # Turn  Needs Review D624: No bug, Automated HSTS ...
   # into D624
   for diff in $($ARC list | grep "Needs Review" | grep -E "Automated HSTS|Automated HPKP|Automated blocklist" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH)  }')
   do
     echo "Removing old request $diff"
     # There is no 'arc abandon', see bug 1452082
     echo '{"transactions": [{"type":"abandon"}], "objectIdentifier": "'"${diff}"'"}' | arc call-conduit differential.revision.edit
@@ -435,16 +424,17 @@ while [ $# -gt 0 ]; do
     -n) DRY_RUN=true ;;
     -c) CLOSED_TREE=true ;;
     -d) DONTBUILD=true ;;
     -a) APPROVAL=true ;;
     --pinset) DO_PRELOAD_PINSET=true ;;
     --hsts) DO_HSTS=true ;;
     --hpkp) DO_HPKP=true ;;
     --blocklist) DO_BLOCKLIST=true ;;
+    --remote-settings) DO_REMOTE_SETTINGS=true ;;
     -r) REPODIR="$2"; shift ;;
     --use-mozilla-central) USE_MC=true ;;
     --use-ftp-builds) USE_TC=false ;;
     -*) usage
       exit 11 ;;
     *)  break ;; # terminate while loop
   esac
   shift
@@ -453,19 +443,19 @@ done
 # Must supply a code branch to work with.
 if [ "${BRANCH}" == "" ]; then
   echo "Error: You must specify a branch with -b branchname." >&2
   usage
   exit 12
 fi
 
 # Must choose at least one update action.
-if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ]
+if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ]
 then
-  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist" >&2
+  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist, --remote-settings" >&2
   usage
   exit 13
 fi
 
 # per-product constants
 case "${PRODUCT}" in
   thunderbird)
     APP_DIR="mail"
@@ -483,39 +473,45 @@ case "${PRODUCT}" in
     exit 14
     ;;
 esac
 
 if [ "${REPODIR}" == "" ]; then
   REPODIR="$(basename "${BRANCH}")"
 fi
 
-HGREPO="https://${HGHOST}/${BRANCH}"
+if [ "${BRANCH}" == "mozilla-central" ]; then
+  HGREPO="https://${HGHOST}/${BRANCH}"
+elif [[ "${BRANCH}" == mozilla-* ]]; then
+  HGREPO="https://${HGHOST}/releases/${BRANCH}"
+else
+  HGREPO="https://${HGHOST}/projects/${BRANCH}"
+fi
 MCREPO="https://${HGHOST}/mozilla-central"
 
 # Remove once 52esr is off support
 VERSION=$(get_version "${HGREPO}")
 MAJOR_VERSION="${VERSION%.*}"
 echo "INFO: parsed version is ${VERSION}"
 if [ "${USE_MC}" == "true" ]; then
   MCVERSION=$(get_version "${MCREPO}")
   echo "INFO: parsed mozilla-central version is ${MCVERSION}"
   MAJOR_VERSION="${MCVERSION%.*}"
 fi
 
 BROWSER_ARCHIVE="${PRODUCT}-${VERSION}.en-US.${PLATFORM}.${PLATFORM_EXT}"
 TESTS_ARCHIVE="${PRODUCT}-${VERSION}.en-US.${PLATFORM}.common.tests.zip"
 if [ "${USE_MC}" == "true" ]; then
-    BROWSER_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.${PLATFORM_EXT}"
-    TESTS_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.common.tests.zip"
+  BROWSER_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.${PLATFORM_EXT}"
+  TESTS_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.common.tests.zip"
 fi
 # Simple name builds on >=53.0.0
 if [ "${MAJOR_VERSION}" -ge 53 ] ; then
-    BROWSER_ARCHIVE="target.${PLATFORM_EXT}"
-    TESTS_ARCHIVE="target.common.tests.zip"
+  BROWSER_ARCHIVE="target.${PLATFORM_EXT}"
+  TESTS_ARCHIVE="target.common.tests.zip"
 fi
 # End 'remove once 52esr is off support'
 
 preflight_cleanup
 if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ]
 then
   if [ "${USE_TC}" == "true" ]; then
     download_shared_artifacts_from_tc
@@ -538,49 +534,72 @@ if [ "${DO_HPKP}" == "true" ]; then
   fi
 fi
 if [ "${DO_BLOCKLIST}" == "true" ]; then
   if compare_blocklist_files
   then
     BLOCKLIST_UPDATED=true
   fi
 fi
+if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
+  if compare_remote_settings_files
+  then
+    REMOTE_SETTINGS_UPDATED=true
+  fi
+fi
 
-if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ]; then
+if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ]; then
   echo "INFO: no updates required. Exiting."
   exit 0
 else
   if [ "${DRY_RUN}" == "true" ]; then
     echo "INFO: Updates are available, not updating hg in dry-run mode."
     exit 2
   fi
 fi
 
 # Currently less reliable than regular 'hg'
 # clone_build_tools
 
 clone_repo
 
-MUST_PUSH=false
+COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update"
 if [ "${HSTS_UPDATED}" == "true" ]
 then
-  commit_hsts_files
-  MUST_PUSH=true
+  stage_hsts_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} HSTS"
 fi
 
 if [ "${HPKP_UPDATED}" == "true" ]
 then
-  commit_hpkp_files
-  MUST_PUSH=true
+  stage_hpkp_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP"
 fi
 
 if [ "${BLOCKLIST_UPDATED}" == "true" ]
 then
-  commit_blocklist_files
-  MUST_PUSH=true
+  stage_blocklist_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} blocklist"
+fi
+
+if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
+then
+  stage_remote_settings_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings"
 fi
 
-if [ -n "${MUST_PUSH}" ]
+if [ ${DONTBUILD} == true ]; then
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
+fi
+if [ ${CLOSED_TREE} == true ]; then
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
+fi
+if [ ${APPROVAL} == true ]; then
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=repo-update"
+fi
+
+
+if ${HG} -R "${REPODIR}" commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 then
   push_repo
 fi
 
 echo "All done"