client.mk
author Mike Hommey <mh+mozilla@glandium.org>
Wed, 24 Jan 2018 11:18:13 +0900
changeset 455913 d7d6780f2b42e0a434803bb2a734c7b8564713a1
parent 446512 249a8177ad915734b83c357d49213e26d889b377
child 470276 da98166e73a816cb3973c799e1e9c46ec2bd9dc7
permissions -rw-r--r--
Bug 1432390 - Use zstandard and requests modules instead of spawning curl | zstd in docker.load_image. r=dustin The zstd command we spawn, if available at all, might be the wrong version: zstd changed its stream format in an incompatible way at some point, and the version shipped in e.g. Ubuntu 16.04 uses the old format, while the version taskcluster relies on uses the new format. Relying on gps's zstandard library allows to ensure we use the right version. Another advantage is that we can trivially pip install it in a virtualenv if it isn't available on the system running the command. If we're ridding ourselves of the subprocess spawning for zstd, we might as well cover curl as well. Especially considering the error handling when subprocesses are involved is not trivial, such that the current error handling code is actually broken and leads to dead-lock conditions, when, for example, curl is still waiting for the python side to read data, but the python side is not reading data anymore because an exception was thrown in the tar reading loop.

# -*- makefile -*-
# vim:set ts=8 sw=8 sts=8 noet:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Defines main targets for driving the Firefox build system.
#
# This make file should not be invoked directly. Instead, use
# `mach` (likely `mach build`) for invoking the build system.
#
# Options:
#   MOZ_OBJDIR           - Destination object directory
#   MOZ_MAKE_FLAGS       - Flags to pass to $(MAKE)
#
#######################################################################
# Defines

ifdef MACH
ifndef NO_BUILDSTATUS_MESSAGES
define BUILDSTATUS
@echo 'BUILDSTATUS $1'

endef
endif
endif


CWD := $(CURDIR)

ifeq "$(CWD)" "/"
CWD   := /.
endif

PYTHON ?= $(shell which python2.7 > /dev/null 2>&1 && echo python2.7 || echo python)

####################################
# Load mozconfig Options

include $(OBJDIR)/.mozconfig-client-mk

ifdef MOZ_PARALLEL_BUILD
  MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
  MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
endif

# Automatically add -jN to make flags if not defined. N defaults to number of cores.
ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
  cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
  MOZ_MAKE_FLAGS += -j$(cores)
endif

ifdef MOZ_AUTOMATION
ifeq (4.0,$(firstword $(sort 4.0 $(MAKE_VERSION))))
MOZ_MAKE_FLAGS += --output-sync=line
endif
endif

MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR)

# 'configure' scripts generated by autoconf.
CONFIGURES := $(TOPSRCDIR)/configure
CONFIGURES += $(TOPSRCDIR)/js/src/configure

#######################################################################
# Rules

# The default rule is build
build::

ifndef MACH
$(error client.mk must be used via `mach`. Try running \
`./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
endif

# In automation, manage an sccache daemon. The starting of the server
# needs to be in a make file so sccache inherits the jobserver.
ifdef MOZBUILD_MANAGE_SCCACHE_DAEMON
build::
	# Terminate any sccache server that might still be around.
	-$(MOZBUILD_MANAGE_SCCACHE_DAEMON) --stop-server > /dev/null 2>&1
	# Start a new server, ensuring it gets the jobserver file descriptors
	# from make (but don't use the + prefix when make -n is used, so that
	# the command doesn't run in that case)
	$(if $(findstring n,$(filter-out --%, $(MAKEFLAGS))),,+)env RUST_LOG=sccache::compiler=debug SCCACHE_ERROR_LOG=$(OBJDIR)/dist/sccache.log $(MOZBUILD_MANAGE_SCCACHE_DAEMON) --start-server
endif

####################################
# Configure

MAKEFILE      = $(wildcard $(OBJDIR)/Makefile)
CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status)

EXTRA_CONFIG_DEPS := \
  $(TOPSRCDIR)/aclocal.m4 \
  $(TOPSRCDIR)/old-configure.in \
  $(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
  $(TOPSRCDIR)/js/src/aclocal.m4 \
  $(TOPSRCDIR)/js/src/old-configure.in \
  $(NULL)

$(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
	@echo Generating $@
	cp -f $< $@
	chmod +x $@

CONFIG_STATUS_DEPS := \
  $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
  $(CONFIGURES) \
  $(TOPSRCDIR)/nsprpub/configure \
  $(TOPSRCDIR)/config/milestone.txt \
  $(TOPSRCDIR)/browser/config/version.txt \
  $(TOPSRCDIR)/browser/config/version_display.txt \
  $(TOPSRCDIR)/build/virtualenv_packages.txt \
  $(TOPSRCDIR)/python/mozbuild/mozbuild/virtualenv.py \
  $(TOPSRCDIR)/testing/mozbase/packages.txt \
  $(OBJDIR)/.mozconfig.json \
  $(NULL)

# Include a dep file emitted by configure to track Python files that
# may influence the result of configure.
-include $(OBJDIR)/configure.d

CONFIGURE_ENV_ARGS += \
  MAKE='$(MAKE)' \
  $(NULL)

# configure uses the program name to determine @srcdir@. Calling it without
#   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
#   path of $(TOPSRCDIR).
ifeq ($(TOPSRCDIR),$(OBJDIR))
  CONFIGURE = ./configure
else
  CONFIGURE = $(TOPSRCDIR)/configure
endif

configure-files: $(CONFIGURES)

configure-preqs = \
  configure-files \
  $(OBJDIR)/.mozconfig.json \
  $(NULL)

configure:: $(configure-preqs)
	$(call BUILDSTATUS,TIERS configure)
	$(call BUILDSTATUS,TIER_START configure)
	@echo cd $(OBJDIR);
	@echo $(CONFIGURE) $(CONFIGURE_ARGS)
	@cd $(OBJDIR) && $(CONFIGURE_ENV_ARGS) $(CONFIGURE) $(CONFIGURE_ARGS) \
	  || ( echo '*** Fix above errors and then restart with\
               "$(MAKE) -f client.mk build"' && exit 1 )
	@touch $(OBJDIR)/Makefile
	$(call BUILDSTATUS,TIER_FINISH configure)

ifneq (,$(MAKEFILE))
$(OBJDIR)/Makefile: $(OBJDIR)/config.status

$(OBJDIR)/config.status: $(CONFIG_STATUS_DEPS)
else
$(OBJDIR)/Makefile: $(CONFIG_STATUS_DEPS)
endif
	@$(MAKE) -f $(TOPSRCDIR)/client.mk configure

####################################
# Build it

build::  $(OBJDIR)/Makefile $(OBJDIR)/config.status
	+$(MOZ_MAKE)

ifdef MOZ_AUTOMATION
build::
	+$(MOZ_MAKE) automation/build
endif

ifdef MOZBUILD_MANAGE_SCCACHE_DAEMON
build::
	# Terminate sccache server. This prints sccache stats.
	-$(MOZBUILD_MANAGE_SCCACHE_DAEMON) --stop-server
endif

# This makefile doesn't support parallel execution. It does pass
# MOZ_MAKE_FLAGS to sub-make processes, so they will correctly execute
# in parallel.
.NOTPARALLEL:

.PHONY: \
    build \
    configure