Split and extend bsd.test.mk into {atf,plain,tap}.test.mk.

This is a MFC of:

- r256761 Clearly split the logic to build ATF and plain tests apart.
- r256762 Add the automatic generation of Atffile files.
- r256763 Add the automatic generation of Kyuafile files.
- r256764 Plug atf-run into the 'test' target.
- r256765 Plug kyua into the 'test' target.
- r257096 Move the TESTSBASE definition to bsd.own.mk.
- r257099 Add missing plain.test.mk.
- r258297 Remove registration of C++ test programs into PROGS.
- r258298 Fix the build of plain test programs.
- r258551 Install plain.test.mk.
- r259208 Add tap.test.mk.

Approved by:	rpaulo (mentor)

7 files changed, 385 insertions, 50 deletions

diff --git a/lib/libcrypt/tests/Makefile b/lib/libcrypt/tests/Makefile
index 3190dbe..8c818bd 100644
--- a/lib/libcrypt/tests/Makefile
+++ b/lib/libcrypt/tests/Makefile
@@ -2,7 +2,7 @@
 
 # exercise libcrypt
 
-TESTS_C= crypt_tests
+ATF_TESTS_C= crypt_tests
 
 CFLAGS+= -I${.CURDIR:H}
 LDADD+= -L${.OBJDIR:H} -lcrypt
diff --git a/share/mk/Makefile b/share/mk/Makefile
index 2110b26..d07402b 100644
--- a/share/mk/Makefile
+++ b/share/mk/Makefile
@@ -45,6 +45,8 @@ FILESDIR=	${BINDIR}/mk
 
 .if ${MK_ATF} != "no"
 FILES+=	atf.test.mk
+FILES+=	plain.test.mk
+FILES+=	tap.test.mk
 .endif
 
 .include <bsd.prog.mk>
diff --git a/share/mk/atf.test.mk b/share/mk/atf.test.mk
index da4d12f..3ffb2cb 100644
--- a/share/mk/atf.test.mk
+++ b/share/mk/atf.test.mk
@@ -1,49 +1,99 @@
-# $NetBSD$
 # $FreeBSD$
 #
+# Logic to build and install ATF test programs; i.e. test programs linked
+# against the ATF libraries.
 
 .include <bsd.init.mk>
 
-ATF_TESTS:=
+# List of C, C++ and shell test programs to build.
+#
+# Programs listed here are built using PROGS, PROGS_CXX and SCRIPTS,
+# respectively, from bsd.prog.mk.  However, the build rules are tweaked to
+# require the ATF libraries.
+#
+# Test programs registered in this manner are set to be installed into TESTSDIR
+# (which should be overriden by the Makefile) and are not required to provide a
+# manpage.
+ATF_TESTS_C?=
+ATF_TESTS_CXX?=
+ATF_TESTS_SH?=
 
-.if make(*test)
-TESTSDIR?= .
-.endif
+# Whether to allow using the deprecated ATF tools or not.
+#
+# If 'yes', this file will generate Atffiles when requested and will also
+# support using the deprecated atf-run tool to execute the tests.
+ALLOW_DEPRECATED_ATF_TOOLS?= no
 
-.if defined(ATF_TESTS_SUBDIRS)
-# Only visit subdirs when building, etc because ATF does this it on its own.
-.if !make(atf-test)
-SUBDIR+= ${ATF_TESTS_SUBDIRS}
-.endif
-ATF_TESTS+= ${ATF_TESTS_SUBDIRS}
+# Knob to control the handling of the Atffile for this Makefile.
+#
+# If 'yes', an Atffile exists in the source tree and is installed into
+# TESTSDIR.
+#
+# If 'auto', an Atffile is automatically generated based on the list of test
+# programs built by the Makefile and is installed into TESTSDIR.  This is the
+# default and is sufficient in the majority of the cases.
+#
+# If 'no', no Atffile is installed.
+ATFFILE?= auto
 
-.include <bsd.subdir.mk>
-.endif
+# Path to the prefix of the installed ATF tools, if any.
+#
+# If atf-run and atf-report are installed from ports, we automatically define a
+# realtest target below to run the tests using these tools.  The tools are
+# searched for in the hierarchy specified by this variable.
+ATF_PREFIX?= /usr/local
+
+# C compiler passed to ATF tests that need to build code.
+ATF_BUILD_CC?= ${DESTDIR}/usr/bin/cc
+TESTS_ENV+= ATF_BUILD_CC=${ATF_BUILD_CC}
+
+# C preprocessor passed to ATF tests that need to build code.
+ATF_BUILD_CPP?= ${DESTDIR}/usr/bin/cpp
+TESTS_ENV+= ATF_BUILD_CPP=${ATF_BUILD_CPP}
+
+# C++ compiler passed to ATF tests that need to build code.
+ATF_BUILD_CXX?= ${DESTDIR}/usr/bin/c++
+TESTS_ENV+= ATF_BUILD_CXX=${ATF_BUILD_CXX}
 
-.if defined(TESTS_C)
-ATF_TESTS+= ${TESTS_C}
-.for _T in ${TESTS_C}
+# Shell interpreter used to run atf-sh(1) based tests.
+ATF_SHELL?= ${DESTDIR}/bin/sh
+TESTS_ENV+= ATF_SHELL=${ATF_SHELL}
+
+.if !empty(ATF_TESTS_C)
+PROGS+= ${ATF_TESTS_C}
+_TESTS+= ${ATF_TESTS_C}
+.for _T in ${ATF_TESTS_C}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
 SRCS.${_T}?= ${_T}.c
 DPADD.${_T}+= ${LIBATF_C}
 LDADD.${_T}+= -latf-c
+TEST_INTERFACE.${_T}= atf
 .endfor
 .endif
 
-.if defined(TESTS_CXX)
-ATF_TESTS+= ${TESTS_CXX}
-.for _T in ${TESTS_CXX}
+.if !empty(ATF_TESTS_CXX)
+PROGS_CXX+= ${ATF_TESTS_CXX}
+_TESTS+= ${ATF_TESTS_CXX}
+.for _T in ${ATF_TESTS_CXX}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
 SRCS.${_T}?= ${_T}${CXX_SUFFIX:U.cc}
 DPADD.${_T}+= ${LIBATF_CXX} ${LIBATF_C}
 LDADD.${_T}+= -latf-c++ -latf-c
+TEST_INTERFACE.${_T}= atf
 .endfor
 .endif
 
-.if defined(TESTS_SH)
-ATF_TESTS+= ${TESTS_SH}
-.for _T in ${TESTS_SH}
+.if !empty(ATF_TESTS_SH)
+SCRIPTS+= ${ATF_TESTS_SH}
+_TESTS+= ${ATF_TESTS_SH}
+.for _T in ${ATF_TESTS_SH}
+SCRIPTSDIR_${_T}= ${TESTSDIR}
+TEST_INTERFACE.${_T}= atf
 CLEANFILES+= ${_T} ${_T}.tmp
-TESTS_SH_SRC_${_T}?= ${_T}.sh
-${_T}: ${TESTS_SH_SRC_${_T}}
+ATF_TESTS_SH_SRC_${_T}?= ${_T}.sh
+${_T}: ${ATF_TESTS_SH_SRC_${_T}}
 	echo '#! /usr/bin/atf-sh' > ${.TARGET}.tmp
 	cat ${.ALLSRC} >> ${.TARGET}.tmp
 	chmod +x ${.TARGET}.tmp
@@ -51,4 +101,68 @@ ${_T}: ${TESTS_SH_SRC_${_T}}
 .endfor
 .endif
 
+.if ${ALLOW_DEPRECATED_ATF_TOOLS} != "no"
+
+.if ${ATFFILE:tl} != "no"
+FILES+=	Atffile
+FILESDIR_Atffile= ${TESTSDIR}
+
+.if ${ATFFILE:tl} == "auto"
+CLEANFILES+= Atffile Atffile.tmp
+
+Atffile: Makefile
+	@{ echo 'Content-Type: application/X-atf-atffile; version="1"'; \
+	echo; \
+	echo '# Automatically generated by atf-test.mk.'; \
+	echo; \
+	echo 'prop: test-suite = "'${TESTSUITE}'"'; \
+	echo; \
+	for tp in ${ATF_TESTS_C} ${ATF_TESTS_CXX} ${ATF_TESTS_SH} \
+	    ${TESTS_SUBDIRS}; \
+	do \
+	    echo "tp: $${tp}"; \
+	done; } >Atffile.tmp
+	@mv Atffile.tmp Atffile
+.endif
+.endif
+
+ATF_REPORT?= ${ATF_PREFIX}/bin/atf-report
+ATF_RUN?= ${ATF_PREFIX}/bin/atf-run
+.if exists(${ATF_RUN}) && exists(${ATF_REPORT})
+# Definition of the "make test" target and supporting variables.
+#
+# This target, by necessity, can only work for native builds (i.e. a freeBSD
+# host building a release for the same system).  The target runs ATF, which is
+# not in the toolchain, and the tests execute code built for the target host.
+#
+# Due to the dependencies of the binaries built by the source tree and how they
+# are used by tests, it is highly possible for a execution of "make test" to
+# report bogus results unless the new binaries are put in place.
+_TESTS_FIFO= ${.OBJDIR}/atf-run.fifo
+_TESTS_LOG= ${.OBJDIR}/atf-run.log
+CLEANFILES+= ${_TESTS_FIFO} ${_TESTS_LOG}
+realtest: .PHONY
+	@set -e; \
+	if [ -z "${TESTSDIR}" ]; then \
+	    echo "*** No TESTSDIR defined; nothing to do."; \
+	    exit 0; \
+	fi; \
+	cd ${DESTDIR}${TESTSDIR}; \
+	rm -f ${_TESTS_FIFO}; \
+	mkfifo ${_TESTS_FIFO}; \
+	tee ${_TESTS_LOG} < ${_TESTS_FIFO} | ${TESTS_ENV} ${ATF_REPORT} & \
+	set +e; \
+	${TESTS_ENV} ${ATF_RUN} >> ${_TESTS_FIFO}; \
+	result=$${?}; \
+	wait; \
+	rm -f ${_TESTS_FIFO}; \
+	echo; \
+	echo "*** The verbatim output of atf-run has been saved to ${_TESTS_LOG}"; \
+	echo "***"; \
+	echo "*** WARNING: atf-run is deprecated; please install kyua instead"; \
+	exit $${result}
+.endif
+
+.endif
+
 .include <bsd.test.mk>
diff --git a/share/mk/bsd.own.mk b/share/mk/bsd.own.mk
index e2f9894..972a526 100644
--- a/share/mk/bsd.own.mk
+++ b/share/mk/bsd.own.mk
@@ -659,4 +659,8 @@ $xGRP=	${_gid}
 
 .endif # !_WITHOUT_SRCCONF
 
+# Pointer to the top directory into which tests are installed.  Should not be
+# overriden by Makefiles, but the user may choose to set this in src.conf(5).
+TESTSBASE?= /usr/tests
+
 .endif	# !target(__<bsd.own.mk>__)
diff --git a/share/mk/bsd.test.mk b/share/mk/bsd.test.mk
index a75298f..8cdf914 100644
--- a/share/mk/bsd.test.mk
+++ b/share/mk/bsd.test.mk
@@ -1,33 +1,73 @@
-# $NetBSD: bsd.test.mk,v 1.21 2012/08/25 22:21:16 jmmv Exp $
 # $FreeBSD$
+#
+# Generic build infrastructure for test programs.
+#
+# The code in this file is independent of the implementation of the test
+# programs being built; this file just provides generic infrastructure for the
+# build and the definition of various helper variables and targets.
+#
+# Makefiles should never include this file directly.  Instead, they should
+# include one of the various *.test.mk depending on the specific test programs
+# being built.
 
 .include <bsd.init.mk>
 
-.if defined(TESTS_C)
-PROGS+=	${TESTS_C}
-.for _T in ${TESTS_C}
-BINDIR.${_T}= ${TESTSDIR}
-MAN.${_T}?= # empty
-.endfor
-.endif
+# Directory in which to install tests defined by the current Makefile.
+# Makefiles have to override this to point to a subdirectory of TESTSBASE.
+TESTSDIR?= .
 
-.if defined(TESTS_CXX)
-PROGS_CXX+= ${TESTS_CXX}
-PROGS+= ${TESTS_CXX}
-.for _T in ${TESTS_CXX}
-BINDIR.${_T}= ${TESTSDIR}
-MAN.${_T}?= # empty
-.endfor
-.endif
+# Name of the test suite these tests belong to.  Should rarely be changed for
+# Makefiles built into the FreeBSD src tree.
+TESTSUITE?= FreeBSD
 
-.if defined(TESTS_SH)
-SCRIPTS+= ${TESTS_SH}
-.for _T in ${TESTS_SH}
-SCRIPTSDIR_${_T}= ${TESTSDIR}
-.endfor
-.endif
+# List of subdirectories containing tests into which to recurse.  This has the
+# same semantics as SUBDIR at build-time.  However, the directories listed here
+# get registered into the run-time test suite definitions so that the test
+# engines know to recurse into these directories.
+#
+# In other words: list here any directories that contain test programs but use
+# SUBDIR for directories that may contain helper binaries and/or data files.
+TESTS_SUBDIRS?=
+
+# Knob to control the handling of the Kyuafile for this Makefile.
+#
+# If 'yes', a Kyuafile exists in the source tree and is installed into
+# TESTSDIR.
+#
+# If 'auto', a Kyuafile is automatically generated based on the list of test
+# programs built by the Makefile and is installed into TESTSDIR.  This is the
+# default and is sufficient in the majority of the cases.
+#
+# If 'no', no Kyuafile is installed.
+KYUAFILE?= auto
+
+# List of variables to pass to the tests at run-time via the environment.
+TESTS_ENV?=
+
+# Ordered list of directories to construct the PATH for the tests.
+TESTS_PATH+= ${DESTDIR}/bin ${DESTDIR}/sbin \
+             ${DESTDIR}/usr/bin ${DESTDIR}/usr/sbin
+TESTS_ENV+= PATH=${TESTS_PATH:tW:C/ +/:/g}
 
-TESTSBASE?= ${DESTDIR}/usr/tests
+# Ordered list of directories to construct the LD_LIBRARY_PATH for the tests.
+TESTS_LD_LIBRARY_PATH+= ${DESTDIR}/lib ${DESTDIR}/usr/lib
+TESTS_ENV+= LD_LIBRARY_PATH=${TESTS_LD_LIBRARY_PATH:tW:C/ +/:/g}
+
+# List of all tests being built.  This variable is internal should not be
+# defined by the Makefile.  The various *.test.mk modules extend this variable
+# as needed.
+_TESTS?=
+
+# Path to the prefix of the installed Kyua CLI, if any.
+#
+# If kyua is installed from ports, we automatically define a realtest target
+# below to run the tests using this tool.  The tools are searched for in the
+# hierarchy specified by this variable.
+KYUA_PREFIX?= /usr/local
+
+.if !empty(TESTS_SUBDIRS)
+SUBDIR+= ${TESTS_SUBDIRS}
+.endif
 
 # it is rare for test cases to have man pages
 .if !defined(MAN)
@@ -39,8 +79,57 @@ WITHOUT_MAN=yes
 PROG_VARS+= BINDIR
 PROGS_TARGETS+= install
 
-.if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS)
-.include <bsd.progs.mk>
+.if ${KYUAFILE:tl} != "no"
+FILES+=	Kyuafile
+FILESDIR_Kyuafile= ${TESTSDIR}
+
+.if ${KYUAFILE:tl} == "auto"
+CLEANFILES+= Kyuafile Kyuafile.tmp
+
+Kyuafile: Makefile
+	@{ \
+	    echo '-- Automatically generated by bsd.test.mk.'; \
+	    echo; \
+	    echo 'syntax(2)'; \
+	    echo; \
+	    echo 'test_suite("${TESTSUITE}")'; \
+            echo; \
+	} >Kyuafile.tmp
+.for _T in ${_TESTS}
+	@echo "${TEST_INTERFACE.${_T}}_test_program{name=\"${_T}\"}" \
+	    >>Kyuafile.tmp
+.endfor
+.for _T in ${TESTS_SUBDIRS:N.WAIT}
+	@echo "include(\"${_T}/Kyuafile\")" >>Kyuafile.tmp
+.endfor
+	@mv Kyuafile.tmp Kyuafile
+.endif
+.endif
+
+KYUA?= ${KYUA_PREFIX}/bin/kyua
+.if exists(${KYUA})
+# Definition of the "make test" target and supporting variables.
+#
+# This target, by necessity, can only work for native builds (i.e. a FreeBSD
+# host building a release for the same system).  The target runs Kyua, which is
+# not in the toolchain, and the tests execute code built for the target host.
+#
+# Due to the dependencies of the binaries built by the source tree and how they
+# are used by tests, it is highly possible for a execution of "make test" to
+# report bogus results unless the new binaries are put in place.
+realtest: .PHONY
+	@echo "*** WARNING: make test is experimental"
+	@echo "***"
+	@echo "*** Using this test does not preclude you from running the tests"
+	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
+	@echo "*** positives and/or false negatives."
+	@echo
+	@set -e; \
+	${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \
+	result=0; \
+	echo; \
+	echo "*** Once again, note that "make test" is unsupported."; \
+	test $${result} -eq 0
 .endif
 
 beforetest: .PHONY
@@ -76,4 +165,14 @@ test: beforetest realtest
 test: aftertest
 .endif
 
+.if !empty(SUBDIR)
+.include <bsd.subdir.mk>
+.endif
+
+.if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS)
+.include <bsd.progs.mk>
+.elif !empty(FILES)
+.include <bsd.files.mk>
+.endif
+
 .include <bsd.obj.mk>
diff --git a/share/mk/plain.test.mk b/share/mk/plain.test.mk
new file mode 100644
index 0000000..b41c585
--- /dev/null
+++ b/share/mk/plain.test.mk
@@ -0,0 +1,52 @@
+# $FreeBSD$
+#
+# Logic to build and install plain test programs.  A plain test programs it not
+# supposed to use any specific testing framework: all it does is run some code
+# and report the test's pass or fail status via a 0 or 1 exit code.
+
+.include <bsd.init.mk>
+
+# List of C, C++ and shell test programs to build.
+#
+# Programs listed here are built according to the semantics of bsd.prog.mk for
+# PROGS, PROGS_CXX and SCRIPTS, respectively.
+#
+# Test programs registered in this manner are set to be installed into TESTSDIR
+# (which should be overriden by the Makefile) and are not required to provide a
+# manpage.
+PLAIN_TESTS_C?=
+PLAIN_TESTS_CXX?=
+PLAIN_TESTS_SH?=
+
+.if !empty(PLAIN_TESTS_C)
+PROGS+= ${PLAIN_TESTS_C}
+_TESTS+= ${PLAIN_TESTS_C}
+.for _T in ${PLAIN_TESTS_C}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.c
+TEST_INTERFACE.${_T}= plain
+.endfor
+.endif
+
+.if !empty(PLAIN_TESTS_CXX)
+PROGS_CXX+= ${PLAIN_TESTS_CXX}
+_TESTS+= ${PLAIN_TESTS_CXX}
+.for _T in ${PLAIN_TESTS_CXX}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.cc
+TEST_INTERFACE.${_T}= plain
+.endfor
+.endif
+
+.if !empty(PLAIN_TESTS_SH)
+SCRIPTS+= ${PLAIN_TESTS_SH}
+_TESTS+= ${PLAIN_TESTS_SH}
+.for _T in ${PLAIN_TESTS_SH}
+SCRIPTSDIR_${_T}= ${TESTSDIR}
+TEST_INTERFACE.${_T}= plain
+.endfor
+.endif
+
+.include <bsd.test.mk>
diff --git a/share/mk/tap.test.mk b/share/mk/tap.test.mk
new file mode 100644
index 0000000..394f685
--- /dev/null
+++ b/share/mk/tap.test.mk
@@ -0,0 +1,64 @@
+# $FreeBSD$
+#
+# Logic to build and install TAP-compliant test programs.
+#
+# This is provided to support existing tests in the FreeBSD source tree
+# (particularly those coming from tools/regression/) that comply with the
+# Test Anything Protocol.  It should not be used for new tests.
+
+.include <bsd.init.mk>
+
+# List of C, C++ and shell test programs to build.
+#
+# Programs listed here are built according to the semantics of bsd.prog.mk for
+# PROGS, PROGS_CXX and SCRIPTS, respectively.
+#
+# Test programs registered in this manner are set to be installed into TESTSDIR
+# (which should be overriden by the Makefile) and are not required to provide a
+# manpage.
+TAP_TESTS_C?=
+TAP_TESTS_CXX?=
+TAP_TESTS_SH?=
+
+.if !empty(TAP_TESTS_C)
+PROGS+= ${TAP_TESTS_C}
+_TESTS+= ${TAP_TESTS_C}
+.for _T in ${TAP_TESTS_C}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.c
+TEST_INTERFACE.${_T}= tap
+.endfor
+.endif
+
+.if !empty(TAP_TESTS_CXX)
+PROGS_CXX+= ${TAP_TESTS_CXX}
+_TESTS+= ${TAP_TESTS_CXX}
+.for _T in ${TAP_TESTS_CXX}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.cc
+TEST_INTERFACE.${_T}= tap
+.endfor
+.endif
+
+.if !empty(TAP_TESTS_SH)
+SCRIPTS+= ${TAP_TESTS_SH}
+_TESTS+= ${TAP_TESTS_SH}
+.for _T in ${TAP_TESTS_SH}
+SCRIPTSDIR_${_T}= ${TESTSDIR}
+TEST_INTERFACE.${_T}= tap
+CLEANFILES+= ${_T} ${_T}.tmp
+# TODO(jmmv): It seems to me that this SED and SRC functionality should
+# exist in bsd.prog.mk along the support for SCRIPTS.  Move it there if
+# this proves to be useful within the tests.
+TAP_TESTS_SH_SED_${_T}?= # empty
+TAP_TESTS_SH_SRC_${_T}?= ${_T}.sh
+${_T}: ${TAP_TESTS_SH_SRC_${_T}}
+	cat ${.ALLSRC} | sed ${TAP_TESTS_SH_SED_${_T}} >${.TARGET}.tmp
+	chmod +x ${.TARGET}.tmp
+	mv ${.TARGET}.tmp ${.TARGET}
+.endfor
+.endif
+
+.include <bsd.test.mk>