Bug 1467456 Use correct hg repo in repo-update r=lguo
authorSimon Fraser <sfraser@mozilla.com>
Thu, 07 Jun 2018 17:55:50 +0000
changeset 475984 c1197954d1b202b9dfa7f3137cc38d2758fa0bc9
parent 475983 f718887bdbc347451109825b62131eaca9518672
child 475985 68114b4c0d76f086a3a6a6e8106982ef4023bc13
child 476051 5404016c6d164a29c8dccdce58363d5b893af565
push id9374
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:43:20 +0000
treeherdermozilla-beta@160e085dfb0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerslguo
bugs1467456
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1467456 Use correct hg repo in repo-update r=lguo 1. Updated hgrepo to work with mozilla-beta, mozilla-esr60 and project branches (just in case) 2. Presquashed commits, so we only submit one. 3. Replaced 'which' with 'command -v' to avoid future shellcheck issues. Differential Revision: https://phabricator.services.mozilla.com/D1582
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -31,30 +31,30 @@ HG_SSH_USER='ffxbld'
 REPODIR=''
 APP_DIR=''
 APP_ID=''
 APP_NAME=''
 HGHOST="hg.mozilla.org"
 STAGEHOST="archive.mozilla.org"
 WGET="wget -nv"
 UNZIP="unzip -q"
-DIFF="$(which diff) -u"
+DIFF="$(command -v diff) -u"
 BASEDIR="${HOME}"
 TOOLSDIR="${HOME}/tools"
 
 SCRIPTDIR="$(realpath "$(dirname "$0")")"
-HG="$(which hg)"
+HG="$(command -v hg)"
 DATADIR="${BASEDIR}/data"
 mkdir -p "${DATADIR}"
 
 VERSION=''
 MCVERSION=''
 USE_MC=false
 USE_TC=true
-JQ="$(which jq)"
+JQ="$(command -v jq)"
 
 DO_HSTS=false
 HSTS_PRELOAD_SCRIPT="${SCRIPTDIR}/getHSTSPreloadList.js"
 HSTS_PRELOAD_ERRORS="nsSTSPreloadList.errors"
 HSTS_PRELOAD_INC="${DATADIR}/nsSTSPreloadList.inc"
 HSTS_UPDATED=false
 
 DO_HPKP=false
@@ -261,17 +261,17 @@ function compare_hpkp_files {
   then
     return 0
   fi
   return 1
 }
 
 function is_valid_xml {
   xmlfile=$1
-  XMLLINT=$(which xmllint 2>/dev/null | head -n1)
+  XMLLINT=$(command -v xmllint 2>/dev/null | head -n1)
 
   if [ ! -x "${XMLLINT}" ]; then
     echo "ERROR: xmllint not found in PATH"
     exit 60
   fi
   ${XMLLINT} --nonet --noout "${xmlfile}"
 }
 
@@ -307,171 +307,102 @@ function compare_blocklist_files {
 }
 
 function compare_remote_settings_files {
   REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
 
   # 1. List remote settings collections from server.
   echo "INFO: fetch remote settings list from server"
   ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
-      ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
-      # 2. For each entry ${bucket, collection}
-      while IFS="/" read -r bucket collection; do
+    ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
+    # 2. For each entry ${bucket, collection}
+  while IFS="/" read -r bucket collection; do
 
-        # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
-        hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
-        local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
-        mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
-        ${WGET} -qO "$local_location_input" "$hg_dump_url"
-        if [ $? -eq 8 ]; then
-          # We don't keep any dump for this collection, skip it.
-          # Try to clean up in case no collection in this bucket has dump.
-          rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
-          continue
-        fi
+    # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
+    hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
+    local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
+    mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
+    ${WGET} -qO "$local_location_input" "$hg_dump_url"
+    if [ $? -eq 8 ]; then
+      # We don't keep any dump for this collection, skip it.
+      # Try to clean up in case no collection in this bucket has dump.
+      rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
+      continue
+    fi
 
-        # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
-        remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
-        local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
-        mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
-        ${WGET} -qO "$local_location_output" "$remote_records_url"
-      done
+    # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
+    remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
+    local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
+    mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
+    ${WGET} -qO "$local_location_output" "$remote_records_url"
+  done
 
   echo "INFO: diffing old/new remote settings dumps..."
   ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
   if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
   then
     return 0
   fi
   return 1
 }
 
 function clone_build_tools {
-    rm -fr "${TOOLSDIR}"
-    CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
-    ${CLONE_CMD}
+  rm -fr "${TOOLSDIR}"
+  CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
+  ${CLONE_CMD}
 }
 
 # Clones an hg repo
 function clone_repo {
   cd "${BASEDIR}"
   if [ ! -d "${REPODIR}" ]; then
     CLONE_CMD="${HG} clone ${HGREPO} ${REPODIR}"
     ${CLONE_CMD}
   fi
 
   ${HG} -R ${REPODIR} pull
   ${HG} -R ${REPODIR} update -C default
 }
 
 # Copies new HSTS files in place, and commits them.
-function commit_hsts_files {
+function stage_hsts_files {
   cd "${BASEDIR}"
-
   cp -f "${BASEDIR}/${PRODUCT}/$(basename "${HSTS_PRELOAD_INC}")" "${REPODIR}/security/manager/ssl/"
-
-  COMMIT_MESSAGE="No bug, Automated HSTS preload list update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=hsts-update"
-  fi
-  echo "INFO: committing HSTS changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
-# Copies new HPKP files in place, and commits them.
-function commit_hpkp_files {
+function stage_hpkp_files {
   cd "${BASEDIR}"
-
   cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}"
-
-  COMMIT_MESSAGE="No bug, Automated HPKP preload list update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=hpkp-update"
-  fi
-  echo "INFO: committing HPKP changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
-# Copies new blocklist file in place, and commits it.
-function commit_blocklist_files {
+function stage_blocklist_files {
   cd "${BASEDIR}"
   cp -f ${BLOCKLIST_LOCAL_AMO} ${REPODIR}/${APP_DIR}/app/blocklist.xml
-  COMMIT_MESSAGE="No bug, Automated blocklist update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=blocklist-update"
-  fi
-  echo "INFO: committing blocklist changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
-# Copies new remote settings dump files in place, and commits them.
-function commit_remote_settings_files {
+function stage_remote_settings_files {
   cd "${BASEDIR}"
   cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}"
-
-  COMMIT_MESSAGE="No bug, Automated remote settings update"
-  if [ -n "${TASK_ID}" ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
-  fi
-  if [ ${DONTBUILD} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
-  fi
-  if [ ${CLOSED_TREE} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
-  fi
-  if [ ${APPROVAL} == true ]; then
-    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=remote-settings-update"
-  fi
-  echo "INFO: committing remote settings changes"
-  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
 # Push all pending commits to Phabricator
 function push_repo {
   cd "${REPODIR}"
   if [ ! -r "${HOME}/.arcrc" ]
   then
     return 1
   fi
-  if ! ARC=$(which arc)
+  if ! ARC=$(command -v arc)
   then
     return 1
   fi
   if [ -z "${REVIEWERS}" ]
   then
     return 1
   fi
-
   # Clean up older review requests
   # Turn  Needs Review D624: No bug, Automated HSTS ...
   # into D624
   for diff in $($ARC list | grep "Needs Review" | grep -E "Automated HSTS|Automated HPKP|Automated blocklist" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH)  }')
   do
     echo "Removing old request $diff"
     # There is no 'arc abandon', see bug 1452082
     echo '{"transactions": [{"type":"abandon"}], "objectIdentifier": "'"${diff}"'"}' | arc call-conduit differential.revision.edit
@@ -542,39 +473,45 @@ case "${PRODUCT}" in
     exit 14
     ;;
 esac
 
 if [ "${REPODIR}" == "" ]; then
   REPODIR="$(basename "${BRANCH}")"
 fi
 
-HGREPO="https://${HGHOST}/${BRANCH}"
+if [ "${BRANCH}" == "mozilla-central" ]; then
+  HGREPO="https://${HGHOST}/${BRANCH}"
+elif [[ "${BRANCH}" == mozilla-* ]]; then
+  HGREPO="https://${HGHOST}/releases/${BRANCH}"
+else
+  HGREPO="https://${HGHOST}/projects/${BRANCH}"
+fi
 MCREPO="https://${HGHOST}/mozilla-central"
 
 # Remove once 52esr is off support
 VERSION=$(get_version "${HGREPO}")
 MAJOR_VERSION="${VERSION%.*}"
 echo "INFO: parsed version is ${VERSION}"
 if [ "${USE_MC}" == "true" ]; then
   MCVERSION=$(get_version "${MCREPO}")
   echo "INFO: parsed mozilla-central version is ${MCVERSION}"
   MAJOR_VERSION="${MCVERSION%.*}"
 fi
 
 BROWSER_ARCHIVE="${PRODUCT}-${VERSION}.en-US.${PLATFORM}.${PLATFORM_EXT}"
 TESTS_ARCHIVE="${PRODUCT}-${VERSION}.en-US.${PLATFORM}.common.tests.zip"
 if [ "${USE_MC}" == "true" ]; then
-    BROWSER_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.${PLATFORM_EXT}"
-    TESTS_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.common.tests.zip"
+  BROWSER_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.${PLATFORM_EXT}"
+  TESTS_ARCHIVE="${PRODUCT}-${MCVERSION}.en-US.${PLATFORM}.common.tests.zip"
 fi
 # Simple name builds on >=53.0.0
 if [ "${MAJOR_VERSION}" -ge 53 ] ; then
-    BROWSER_ARCHIVE="target.${PLATFORM_EXT}"
-    TESTS_ARCHIVE="target.common.tests.zip"
+  BROWSER_ARCHIVE="target.${PLATFORM_EXT}"
+  TESTS_ARCHIVE="target.common.tests.zip"
 fi
 # End 'remove once 52esr is off support'
 
 preflight_cleanup
 if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ]
 then
   if [ "${USE_TC}" == "true" ]; then
     download_shared_artifacts_from_tc
@@ -619,39 +556,50 @@ else
   fi
 fi
 
 # Currently less reliable than regular 'hg'
 # clone_build_tools
 
 clone_repo
 
-MUST_PUSH=false
+COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update"
 if [ "${HSTS_UPDATED}" == "true" ]
 then
-  commit_hsts_files
-  MUST_PUSH=true
+  stage_hsts_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} HSTS"
 fi
 
 if [ "${HPKP_UPDATED}" == "true" ]
 then
-  commit_hpkp_files
-  MUST_PUSH=true
+  stage_hpkp_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP"
 fi
 
 if [ "${BLOCKLIST_UPDATED}" == "true" ]
 then
-  commit_blocklist_files
-  MUST_PUSH=true
+  stage_blocklist_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} blocklist"
 fi
 
 if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
 then
-  commit_remote_settings_files
-  MUST_PUSH=true
+  stage_remote_settings_files
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings"
 fi
 
-if [ -n "${MUST_PUSH}" ]
+if [ ${DONTBUILD} == true ]; then
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
+fi
+if [ ${CLOSED_TREE} == true ]; then
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
+fi
+if [ ${APPROVAL} == true ]; then
+  COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=repo-update"
+fi
+
+
+if ${HG} -R "${REPODIR}" commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 then
   push_repo
 fi
 
 echo "All done"