Build libcompat (lib32) with a --sysroot pointing into its stage directory.

This overrides the cross-compiler's default sysroot to use the WORLD32's
sysroot for building the lib32 libraries.  Previously the cross-compiler
would default the sysroot to the 64bit WORLDTMP and -B/-L/-isystem flags
were used to build using the lib32 files.  This leads to multiple issues
discussed later.  Some extra headers are now needed to be staged since the
64bit WORLDTMP is not referenced at all for headers.  The 64bit WORLDTMP
is still used via PATH for build tools.  Overriding the default
target/arch is retained in the CC/CXX overrides.

This allows reverting the LDSCRIPT rewriting in installworld from r296921 and
r235122, thus allowing read-only objdirs to work for installing again.

This removes the need for _LDSCRIPTROOT.

This allows progressing the change to always use --sysroot for the build
rather than only relying on the cross-compiler's default sysroot.  The
work for that is in D3970 and needed to resolve WITHOUT_CROSS_COMPILER
not using a --sysroot [1].

PR:		196193 [1]
Sponsored by:	EMC / Isilon Storage Division

4 files changed, 15 insertions, 33 deletions

diff --git a/Makefile.inc1 b/Makefile.inc1
index 077789f..135d9bf 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -360,7 +360,6 @@ KTMAKE=		TOOLS_PREFIX=${WORLDTMP} MAKEOBJDIRPREFIX=${WORLDTMP} \
 
 # world stage
 WMAKEENV=	${CROSSENV} \
-		_LDSCRIPTROOT= \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH}
 
diff --git a/Makefile.libcompat b/Makefile.libcompat
index cf5ea3b..d8845f9 100644
--- a/Makefile.libcompat
+++ b/Makefile.libcompat
@@ -64,13 +64,17 @@ LIBCOMPAT_OBJTREE?=	${OBJTREE}${.CURDIR}/world${libcompat}
 LIBCOMPATTMP?=		${OBJTREE}${.CURDIR}/lib${libcompat}
 
 LIBCOMPATCFLAGS+=	${LIBCOMPATCPUFLAGS} \
-			-isystem ${LIBCOMPATTMP}/usr/include/ \
 			-L${LIBCOMPATTMP}/usr/lib${libcompat} \
-			-B${LIBCOMPATTMP}/usr/lib${libcompat}
+			--sysroot=${LIBCOMPATTMP}
+
+# -B is needed to find /usr/lib32/crti.o for GCC and /usr/libsoft/crti.o for
+# Clang/GCC.
+LIBCOMPATCFLAGS+=	-B${LIBCOMPATTMP}/usr/lib${libcompat}
+# GCC requires -isystem when using a cross-compiler.
+LIBCOMPATCFLAGS+=	-isystem ${LIBCOMPATTMP}/usr/include
 
 # Yes, the flags are redundant.
 LIBCOMPATWMAKEENV+= MAKEOBJDIRPREFIX=${LIBCOMPAT_OBJTREE} \
-		_LDSCRIPTROOT=${LIBCOMPATTMP} \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH} \
 		LIBDIR=/usr/lib${libcompat} \
@@ -89,15 +93,16 @@ LIBCOMPATIMAKE+=	${LIBCOMPATWMAKE:NINSTALL=*:NDESTDIR=*:N_LDSCRIPTROOT=*} \
 			MK_TOOLCHAIN=no ${IMAKE_INSTALL} \
 			-DLIBRARIES_ONLY
 
-.if ${XCC:N${CCACHE_BIN}:M/*}
-LIBCOMPATCFLAGS+=	--sysroot=${WORLDTMP}
-.endif
-
 _LC_LIBDIRS.yes=		lib gnu/lib
 _LC_LIBDIRS.${MK_CDDL:tl}+=	cddl/lib
 _LC_LIBDIRS.${MK_CRYPT:tl}+=	secure/lib
 _LC_LIBDIRS.${MK_KERBEROS:tl}+=	kerberos5/lib
 
+_LC_INCDIRS=	\
+		include \
+		lib/ncurses/ncursesw \
+		${_LC_LIBDIRS.yes}
+
 # Shared logic
 build${libcompat}: .PHONY
 	@echo
@@ -120,8 +125,7 @@ build${libcompat}: .PHONY
 	mkdir -p ${WORLDTMP}
 	ln -sf ${.CURDIR}/sys ${WORLDTMP}
 .for _t in obj includes
-	${_+_}cd ${.CURDIR}/include; ${LIBCOMPATWMAKE} DIRPRFX=include/ ${_t}
-.for _dir in ${_LC_LIBDIRS.yes}
+.for _dir in ${_LC_INCDIRS}
 	${_+_}cd ${.CURDIR}/${_dir}; ${LIBCOMPATWMAKE} DIRPRFX=${_dir}/ ${_t}
 .endfor
 .endfor
diff --git a/share/mk/bsd.lib.mk b/share/mk/bsd.lib.mk
index 4a8fe89..3339ae2 100644
--- a/share/mk/bsd.lib.mk
+++ b/share/mk/bsd.lib.mk
@@ -215,26 +215,10 @@ ${SHLIB_NAME_FULL}: beforelinking
 .endif
 
 .if defined(SHLIB_LINK)
-# ${_LDSCRIPTROOT} is needed when cross-building
-# and when building 32 bits library shims.
-#
-# ${_LDSCRIPTROOT} is the directory prefix that will be used when generating
-# ld(1) scripts.  The crosstools' ld is configured to lookup libraries in an
-# alternative directory which is called "sysroot", so during buildworld binaries
-# won't be linked against the running system libraries but against the ones of
-# the current source tree.  ${_LDSCRIPTROOT} behavior is twisted because of
-# the location where we store them:
-# - 64 bits libs are located under sysroot, so ${_LDSCRIPTROOT} must be empty
-#   because ld(1) will manage to find them from sysroot;
-# - 32 bits shims are not, so ${_LDSCRIPTROOT} is used to specify their full
-#   path, outside of sysroot.
-# Note that ld(1) scripts are generated both during buildworld and
-# installworld; in the later case ${_LDSCRIPTROOT} must be obviously empty
-# because on the target system, libraries are meant to be looked up from /.
 .if defined(SHLIB_LDSCRIPT) && !empty(SHLIB_LDSCRIPT) && exists(${.CURDIR}/${SHLIB_LDSCRIPT})
 ${SHLIB_LINK:R}.ld: ${.CURDIR}/${SHLIB_LDSCRIPT}
-	sed -e 's,@@SHLIB@@,${_LDSCRIPTROOT}${_SHLIBDIR}/${SHLIB_NAME},g' \
-	    -e 's,@@LIBDIR@@,${_LDSCRIPTROOT}${_LIBDIR},g' \
+	sed -e 's,@@SHLIB@@,${_SHLIBDIR}/${SHLIB_NAME},g' \
+	    -e 's,@@LIBDIR@@,${_LIBDIR},g' \
 	    ${.ALLSRC} > ${.TARGET}
 
 ${SHLIB_NAME_FULL}: ${SHLIB_LINK:R}.ld
@@ -338,9 +322,6 @@ _SHLINSTALLFLAGS:=	${_SHLINSTALLFLAGS${ie}}
 .if !defined(INTERNALLIB)
 realinstall: _libinstall
 .ORDER: beforeinstall _libinstall
-.if target(${SHLIB_LINK:R}.ld)
-_libinstall: ${SHLIB_LINK:R}.ld
-.endif
 _libinstall:
 .if defined(LIB) && !empty(LIB) && ${MK_INSTALLLIB} != "no"
 	${INSTALL} -C -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
@@ -367,7 +348,6 @@ _libinstall:
 	${INSTALL} -S -C -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${_INSTALLFLAGS} ${SHLIB_LINK:R}.ld \
 	    ${DESTDIR}${_LIBDIR}/${SHLIB_LINK}
-	rm -f ${SHLIB_LINK:R}.ld
 .for _SHLIB_LINK_LINK in ${SHLIB_LDSCRIPT_LINKS}
 	${INSTALL_SYMLINK} ${SHLIB_LINK} ${DESTDIR}${_LIBDIR}/${_SHLIB_LINK_LINK}
 .endfor
diff --git a/share/mk/bsd.sys.mk b/share/mk/bsd.sys.mk
index 0e6a07ee..d3893d2 100644
--- a/share/mk/bsd.sys.mk
+++ b/share/mk/bsd.sys.mk
@@ -231,7 +231,6 @@ stage_files.shlib: ${_LIBS:M*.so.*}
 .endif
 
 .if defined(SHLIB_LINK) && commands(${SHLIB_LINK:R}.ld)
-#_LDSCRIPTROOT?= ${STAGE_OBJTOP}
 STAGE_AS_SETS+= ldscript
 STAGE_AS.ldscript+= ${SHLIB_LINK:R}.ld
 stage_as.ldscript: ${SHLIB_LINK:R}.ld