Bug 832272 - Allow mostly parallel build of NSS. r=ted
authorMike Hommey <mh+mozilla@glandium.org>
Mon, 25 Feb 2013 10:49:34 +0100
changeset 122894 e81aef9e0d1637b6e787964d5cc2052597fe47f6
parent 122893 6757e2f48c0a465f0b1a8c0e386152c4690f903b
child 122895 179a23a14368dfe49301c56ff32192824a079368
push id1387
push userphilringnalda@gmail.com
push dateTue, 26 Feb 2013 22:32:56 +0000
treeherderfx-team@ad4cc4e97774 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs832272
milestone22.0a1
Bug 832272 - Allow mostly parallel build of NSS. r=ted
security/build/Makefile.in
security/build/b2g-certdata.mk
toolkit/toolkit-tiers.mk
--- a/security/build/Makefile.in
+++ b/security/build/Makefile.in
@@ -6,52 +6,41 @@
 DEPTH		= @DEPTH@
 topsrcdir	= @top_srcdir@
 srcdir		= @srcdir@
 VPATH		= @srcdir@
 
 include $(DEPTH)/config/autoconf.mk
 CC_WRAPPER =
 CXX_WRAPPER =
+
+default::
+
 include $(topsrcdir)/config/config.mk
 
-ifndef MOZ_NATIVE_NSS
-LOADABLE_ROOT_MODULE = $(DLL_PREFIX)nssckbi$(DLL_SUFFIX)
-endif
+NSS_DLLS = \
+  nss3 \
+  nssutil3 \
+  smime3 \
+  ssl3 \
+  $(NULL)
 
-NSS3_LIB = $(DLL_PREFIX)nss3$(DLL_SUFFIX)
-NSSUTIL3_LIB = $(DLL_PREFIX)nssutil3$(DLL_SUFFIX)
-SMIME3_LIB = $(DLL_PREFIX)smime3$(DLL_SUFFIX)
-SSL3_LIB =  $(DLL_PREFIX)ssl3$(DLL_SUFFIX)
-SOFTOKEN3_LIB = $(DLL_PREFIX)softokn3$(DLL_SUFFIX)
+NSS_EXTRA_DLLS = \
+  nssckbi \
+  softokn3 \
+  $(NULL)
 
 ifndef NSS_DISABLE_DBM
-NSSDBM3_LIB = $(DLL_PREFIX)nssdbm3$(DLL_SUFFIX)
-else
-NSSDBM3_LIB = 
+NSS_EXTRA_DLLS += nssdbm3
 endif
 
-ifndef MOZ_NATIVE_NSS
+SDK_LIBS = crmf
+
 ifneq (,$(filter OS2 WINNT,$(OS_ARCH)))
-SDK_LIBS = \
-  $(DIST)/lib/$(LIB_PREFIX)crmf.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)smime3.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)ssl3.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)nss3.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)nssutil3.$(LIB_SUFFIX) \
-  $(NULL)
-else
-SDK_LIBS = \
-  $(DIST)/lib/$(LIB_PREFIX)crmf.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)smime.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)ssl.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)nss.$(LIB_SUFFIX) \
-  $(DIST)/lib/$(LIB_PREFIX)nssutil.$(LIB_SUFFIX) \
-  $(NULL)
-endif
+SDK_LIBS += $(NSS_DLLS)
 endif
 
 # Default
 HAVE_FREEBL_LIBS = 1
 
 # 32-bit HP-UX PA-RISC
 ifeq ($(OS_ARCH), HP-UX)
 ifneq ($(OS_TEST), ia64)
@@ -72,28 +61,28 @@ else
 HAVE_FREEBL_LIBS =
 HAVE_FREEBL_LIBS_32 = 1
 HAVE_FREEBL_LIBS_32INT64 = 1
 endif
 endif
 endif
 
 ifdef HAVE_FREEBL_LIBS
-FREEBL_LIB = $(DLL_PREFIX)freebl3$(DLL_SUFFIX)
+NSS_EXTRA_DLLS += freebl3
 endif
 ifdef HAVE_FREEBL_LIBS_32
-FREEBL_32INT_LIB = libfreebl_32int_3$(DLL_SUFFIX)
-FREEBL_32FPU_LIB = libfreebl_32fpu_3$(DLL_SUFFIX)
+NSS_EXTRA_DLLS += freebl_32int_3
+NSS_EXTRA_DLLS += freebl_32fpu_3
 endif
 ifdef HAVE_FREEBL_LIBS_32INT64
-FREEBL_32INT64_LIB = libfreebl_32int64_3$(DLL_SUFFIX)
+NSS_EXTRA_DLLS += freebl_32int64_3
 endif
 ifdef HAVE_FREEBL_LIBS_64
-FREEBL_64INT_LIB = libfreebl_64int_3$(DLL_SUFFIX)
-FREEBL_64FPU_LIB = libfreebl_64fpu_3$(DLL_SUFFIX)
+NSS_EXTRA_DLLS += freebl_64int_3
+NSS_EXTRA_DLLS += freebl_64fpu_3
 endif
 
 ABS_DIST := $(call core_abspath,$(DIST))
 ifeq ($(HOST_OS_ARCH),WINNT)
 ifdef CYGDRIVE_MOUNT
 ABS_DIST := $(shell cygpath -w $(ABS_DIST) | sed -e 's|\\|/|g')
 endif
 ifneq (,$(filter mingw%,$(host_os)))
@@ -118,28 +107,30 @@ endif
 # definitions in NSS, so just export it into the sub-make's environment.
 ifeq (WINNT_1,$(OS_TARGET)_$(MOZ_MEMORY))
 export DLLFLAGS
 endif
 
 # To get debug symbols from NSS
 export MOZ_DEBUG_SYMBOLS
 
-# NSS makefiles are not safe for parallel execution.
-DEFAULT_GMAKE_FLAGS = -j1
+DEFAULT_GMAKE_FLAGS =
 DEFAULT_GMAKE_FLAGS += CC="$(CC)"
 DEFAULT_GMAKE_FLAGS += SOURCE_MD_DIR=$(ABS_DIST)
 DEFAULT_GMAKE_FLAGS += SOURCE_MDHEADERS_DIR=$(NSPR_INCLUDE_DIR)
 DEFAULT_GMAKE_FLAGS += DIST=$(ABS_DIST)
 DEFAULT_GMAKE_FLAGS += NSPR_INCLUDE_DIR=$(NSPR_INCLUDE_DIR)
 DEFAULT_GMAKE_FLAGS += NSPR_LIB_DIR=$(NSPR_LIB_DIR)
 DEFAULT_GMAKE_FLAGS += MOZILLA_CLIENT=1
 DEFAULT_GMAKE_FLAGS += NO_MDUPDATE=1
 DEFAULT_GMAKE_FLAGS += NSS_ENABLE_ECC=1
 DEFAULT_GMAKE_FLAGS += NSINSTALL="$(NSINSTALL)"
+ifeq ($(OS_ARCH),WINNT)
+DEFAULT_GMAKE_FLAGS += INSTALL="$(NSINSTALL) -t"
+endif
 ifndef MOZ_NATIVE_SQLITE
 DEFAULT_GMAKE_FLAGS += SQLITE_LIB_NAME=mozsqlite3
 DEFAULT_GMAKE_FLAGS += SQLITE_INCLUDE_DIR=$(ABS_DIST)/include
 endif
 ifdef NSS_DISABLE_DBM 
 DEFAULT_GMAKE_FLAGS += NSS_DISABLE_DBM=1
 endif
 ABS_topsrcdir   := $(shell cd $(topsrcdir); pwd)
@@ -246,16 +237,28 @@ endif
 ifdef MOZ_NO_WLZDEFS
 DEFAULT_GMAKE_FLAGS += ZDEFS_FLAG=
 endif
 ifdef MOZ_CFLAGS_NSS
 DEFAULT_GMAKE_FLAGS += XCFLAGS="$(CFLAGS)"
 DEFAULT_GMAKE_FLAGS += DARWIN_DYLIB_VERSIONS="-compatibility_version 1 -current_version 1 $(LDFLAGS)"
 endif
 
+# Put NSS headers directly under $(DIST)/include
+DEFAULT_GMAKE_FLAGS += PUBLIC_EXPORT_DIR='$(ABS_DIST)/include/$$(MODULE)'
+DEFAULT_GMAKE_FLAGS += SOURCE_XPHEADERS_DIR='$$(SOURCE_XP_DIR)/include/$$(MODULE)'
+DEFAULT_GMAKE_FLAGS += MODULE_INCLUDES='$$(addprefix -I$$(SOURCE_XP_DIR)/include/,$$(REQUIRES))'
+
+# Work around NSS's MAKE_OBJDIR being racy. See bug #836220
+DEFAULT_GMAKE_FLAGS += MAKE_OBJDIR='$$(INSTALL) -D $$(OBJDIR)'
+
+# Work around NSS adding IMPORT_LIBRARY to TARGETS with no rule for
+# it, creating race conditions. See bug #836220
+DEFAULT_GMAKE_FLAGS += TARGETS='$$(LIBRARY) $$(SHARED_LIBRARY) $$(PROGRAM)'
+
 ifdef MOZ_B2G_CERTDATA
 include $(srcdir)/b2g-certdata.mk
 endif
 
 ifdef MOZ_NSS_PATCH
 # If we're applying a patch, we'll copy the NSS source to the objdir
 # and build it from there.
 NSS_SRCDIR = $(CURDIR)/nss
@@ -293,58 +296,64 @@ NSS_DIRS += \
   nss/cmd/modutil \
   $(NULL)
 endif
 
 ifneq (,$(filter %--build-id,$(LDFLAGS)))
 DEFAULT_GMAKE_ENV = LDFLAGS=-Wl,--build-id
 endif
 
-ifndef MOZ_NATIVE_NSS
-define build_rules
-libs::
-	$$(DEFAULT_GMAKE_ENV) $$(MAKE) -C $$(NSS_SRCDIR)/security/$(1) $$(DEFAULT_GMAKE_FLAGS)
-
-clean clobber clobber_all realclean distclean depend::
-	$$(MAKE) -C $$(NSS_SRCDIR)/security/$(1) $$(DEFAULT_GMAKE_FLAGS) clean
-endef
-$(foreach dir,$(NSS_DIRS),$(eval $(call build_rules,$(dir))))
+NSS_DIST_DLL_FILES := $(addprefix $(DIST)/lib/$(DLL_PREFIX),$(addsuffix $(DLL_SUFFIX),$(NSS_DLLS) $(NSS_EXTRA_DLLS)))
+NSS_DIST_DLL_DEST := $(DIST)/bin
+INSTALL_TARGETS += NSS_DIST_DLL
 
-NSS_LIBS = \
-  $(LOADABLE_ROOT_MODULE) \
-  $(SOFTOKEN3_LIB) \
-  $(NSSDBM3_LIB) \
-  $(NSS3_LIB) \
-  $(NSSUTIL3_LIB) \
-  $(SSL3_LIB) \
-  $(SMIME3_LIB) \
-  $(FREEBL_LIB) \
-  $(FREEBL_32INT_LIB) \
-  $(FREEBL_32FPU_LIB) \
-  $(FREEBL_32INT64_LIB) \
-  $(FREEBL_64INT_LIB) \
-  $(FREEBL_64FPU_LIB) \
-  $(NULL)
-
-define install_rules
-libs::
-ifeq ($(OS_ARCH)_$(1), SunOS_$(SOFTOKEN3_LIB))
+ifeq ($(OS_ARCH)_$(1), SunOS_softokn3)
 # has to use copy mode on Solaris, see #665509
-	$$(NSINSTALL) -t -m 755 $$(DIST)/lib/$(1) $$(DIST)/bin
-else
-	$$(INSTALL) -m 755 $$(DIST)/lib/$(1) $$(DIST)/bin
+$(DIST)/bin/$(DLL_PREFIX)softokn3$(DLL_SUFFIX): INSTALL := $(INSTALL) -t
 endif
 
-install::
-	$$(SYSINSTALL) -m 755 $$(DIST)/lib/$(1) $$(DESTDIR)$$(mozappdir)
-endef
-$(foreach lib,$(NSS_LIBS),$(eval $(call install_rules,$(lib))))
+NSS_SDK_LIB_FILES := \
+  $(addprefix $(DIST)/lib/$(LIB_PREFIX),$(addsuffix .$(LIB_SUFFIX),$(SDK_LIBS))) \
+  $(addprefix $(DIST)/bin/$(DLL_PREFIX),$(addsuffix $(DLL_SUFFIX),$(NSS_DLLS))) \
+  $(NULL)
+NSS_SDK_LIB_DEST := $(DIST)/sdk/lib
+INSTALL_TARGETS += NSS_SDK_LIB
 
-libs::
-	$(INSTALL) -m 755 $(SDK_LIBS) $(DIST)/sdk/lib
-# NSS installs headers to dist/public and we want them in dist/include
-	$(NSINSTALL) -D $(DIST)/include/nss
-	(cd $(DIST)/public/nss && tar $(TAR_CREATE_FLAGS) - .) | \
-	  (cd $(DIST)/include/nss && tar -xf -)
-
-endif # MOZ_NATIVE_NSS
+$(NSS_DIST_DLL_FILES) $(NSS_SDK_LIB_FILES): libs-nss/lib
 
 include $(topsrcdir)/config/rules.mk
+
+DEFAULT_GMAKE_FLAGS += $(EXTRA_GMAKE_FLAGS)
+
+$(addprefix libs-,$(NSS_DIRS)): libs-%:
+# Work around NSS's export rule being racy when recursing for private_export
+# See bug #836220.
+$(addprefix export-,$(NSS_DIRS)): EXTRA_GMAKE_FLAGS = PRIVATE_EXPORTS=
+$(addprefix export-,$(NSS_DIRS)): export-%: private_export-%
+$(addprefix private_export-,$(NSS_DIRS)): EXTRA_GMAKE_FLAGS =
+$(addprefix private_export-,$(NSS_DIRS)): private_export-%:
+
+# Work around bug #836228 in pymake
+ifdef .PYMAKE
+$(foreach p,libs export private_export,$(addprefix $(p)-,$(NSS_DIRS))): *=$(subst $(NULL) $(NULL),-,$(wordlist 2,$(words $(subst -, ,$@)),$(subst -, ,$@)))
+endif
+
+$(foreach p,libs export private_export,$(addprefix $(p)-,$(NSS_DIRS))):
+	$(DEFAULT_GMAKE_ENV) $(MAKE) -C $(NSS_SRCDIR)/security/$* $(@:-$*=) $(DEFAULT_GMAKE_FLAGS)
+
+export:: $(addprefix export-,$(NSS_DIRS))
+
+$(addprefix clean-,$(NSS_DIRS)): clean-%:
+	$(MAKE) -C $(NSS_SRCDIR)/security/$* $(DEFAULT_GMAKE_FLAGS) clean
+
+clean clobber clobber_all realclean distclean depend:: $(addprefix clean-,$(NSS_DIRS))
+
+NSS_CMD_TARGETS := $(addprefix libs-,$(filter-out nss/cmd/lib,$(filter nss/cmd/%,$(NSS_DIRS))))
+libs:: $(NSS_CMD_TARGETS)
+
+$(NSS_CMD_TARGETS): libs-nss/lib libs-nss/cmd/lib
+ifndef NSS_DISABLE_DBM
+libs-nss/lib: libs-dbm
+endif
+
+# Work around NSS build system race condition creating certdata.c in
+# security/nss/lib/ckfw/builtins. See bug #836220.
+libs-nss/lib: $(call mkdir_deps,../nss/lib/ckfw/builtins)
--- a/security/build/b2g-certdata.mk
+++ b/security/build/b2g-certdata.mk
@@ -25,11 +25,11 @@
 # Distrust all existing builtin CAs for code-signing
 hacked-certdata.txt : $(srcdir)/../nss/lib/ckfw/builtins/certdata.txt
 	sed -e "s/^CKA_TRUST_CODE_SIGNING.*CKT_NSS_TRUSTED_DELEGATOR.*/CKA_TRUST_CODE_SIGNING CK_TRUST CKT_NSS_MUST_VERIFY_TRUST/" \
 			$< > $@
 
 combined-certdata.txt : hacked-certdata.txt $(srcdir)/b2g-certdata.txt
 	cat $^ > $@
 
-libs:: combined-certdata.txt
+libs-nss/lib: combined-certdata.txt
 
 DEFAULT_GMAKE_FLAGS += NSS_CERTDATA_TXT='$(CURDIR)/combined-certdata.txt'
--- a/toolkit/toolkit-tiers.mk
+++ b/toolkit/toolkit-tiers.mk
@@ -8,16 +8,28 @@ endif
 
 TIERS += nspr
 
 ifndef MOZ_NATIVE_NSPR
 tier_nspr_staticdirs += nsprpub
 tier_nspr_dirs += config/nspr
 endif
 
+TIERS += nss
+
+ifndef MOZ_NATIVE_SQLITE
+tier_nss_dirs += db/sqlite3/src
+endif
+
+ifdef MOZ_PSM
+ifndef MOZ_NATIVE_NSS
+tier_nss_dirs += security/build
+endif
+endif
+
 include $(topsrcdir)/config/js/build.mk
 
 TIERS += platform
 
 ifdef NS_TRACE_MALLOC
 tier_platform_dirs = tools/trace-malloc/lib
 endif
 
@@ -26,26 +38,16 @@ tier_platform_dirs += memory/replace/dmd
 endif
 
 ifdef MOZ_TREE_FREETYPE
 tier_platform_staticdirs += modules/freetype2
 endif
 
 tier_platform_dirs += xpcom
 
-ifndef MOZ_NATIVE_SQLITE
-tier_platform_dirs += db/sqlite3/src
-endif
-
-ifdef MOZ_PSM
-tier_platform_dirs += \
-  security/build \
-  $(NULL)
-endif
-
 tier_platform_dirs += \
 		modules/libpref \
 		intl \
 		netwerk \
 		$(NULL)
 
 ifdef MOZ_AUTH_EXTENSION
 tier_platform_dirs += extensions/auth