summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjmmv <jmmv@FreeBSD.org>2013-12-27 22:00:22 +0000
committerjmmv <jmmv@FreeBSD.org>2013-12-27 22:00:22 +0000
commite125f95c9c257e17bc2724cf2f7bdb2a5e1f367c (patch)
tree1119ed400ee859c1a442dea4edd3494fbc8a2f21
parent33f454b5c1f0354b8354b4536b45d3d1d1cc8982 (diff)
downloadFreeBSD-src-e125f95c9c257e17bc2724cf2f7bdb2a5e1f367c.zip
FreeBSD-src-e125f95c9c257e17bc2724cf2f7bdb2a5e1f367c.tar.gz
Split and extend bsd.test.mk into {atf,plain,tap}.test.mk.
This is a MFC of: - r256761 Clearly split the logic to build ATF and plain tests apart. - r256762 Add the automatic generation of Atffile files. - r256763 Add the automatic generation of Kyuafile files. - r256764 Plug atf-run into the 'test' target. - r256765 Plug kyua into the 'test' target. - r257096 Move the TESTSBASE definition to bsd.own.mk. - r257099 Add missing plain.test.mk. - r258297 Remove registration of C++ test programs into PROGS. - r258298 Fix the build of plain test programs. - r258551 Install plain.test.mk. - r259208 Add tap.test.mk. Approved by: rpaulo (mentor)
-rw-r--r--lib/libcrypt/tests/Makefile2
-rw-r--r--share/mk/Makefile2
-rw-r--r--share/mk/atf.test.mk162
-rw-r--r--share/mk/bsd.own.mk4
-rw-r--r--share/mk/bsd.test.mk149
-rw-r--r--share/mk/plain.test.mk52
-rw-r--r--share/mk/tap.test.mk64
7 files changed, 385 insertions, 50 deletions
diff --git a/lib/libcrypt/tests/Makefile b/lib/libcrypt/tests/Makefile
index 3190dbe..8c818bd 100644
--- a/lib/libcrypt/tests/Makefile
+++ b/lib/libcrypt/tests/Makefile
@@ -2,7 +2,7 @@
# exercise libcrypt
-TESTS_C= crypt_tests
+ATF_TESTS_C= crypt_tests
CFLAGS+= -I${.CURDIR:H}
LDADD+= -L${.OBJDIR:H} -lcrypt
diff --git a/share/mk/Makefile b/share/mk/Makefile
index 2110b26..d07402b 100644
--- a/share/mk/Makefile
+++ b/share/mk/Makefile
@@ -45,6 +45,8 @@ FILESDIR= ${BINDIR}/mk
.if ${MK_ATF} != "no"
FILES+= atf.test.mk
+FILES+= plain.test.mk
+FILES+= tap.test.mk
.endif
.include <bsd.prog.mk>
diff --git a/share/mk/atf.test.mk b/share/mk/atf.test.mk
index da4d12f..3ffb2cb 100644
--- a/share/mk/atf.test.mk
+++ b/share/mk/atf.test.mk
@@ -1,49 +1,99 @@
-# $NetBSD$
# $FreeBSD$
#
+# Logic to build and install ATF test programs; i.e. test programs linked
+# against the ATF libraries.
.include <bsd.init.mk>
-ATF_TESTS:=
+# List of C, C++ and shell test programs to build.
+#
+# Programs listed here are built using PROGS, PROGS_CXX and SCRIPTS,
+# respectively, from bsd.prog.mk. However, the build rules are tweaked to
+# require the ATF libraries.
+#
+# Test programs registered in this manner are set to be installed into TESTSDIR
+# (which should be overriden by the Makefile) and are not required to provide a
+# manpage.
+ATF_TESTS_C?=
+ATF_TESTS_CXX?=
+ATF_TESTS_SH?=
-.if make(*test)
-TESTSDIR?= .
-.endif
+# Whether to allow using the deprecated ATF tools or not.
+#
+# If 'yes', this file will generate Atffiles when requested and will also
+# support using the deprecated atf-run tool to execute the tests.
+ALLOW_DEPRECATED_ATF_TOOLS?= no
-.if defined(ATF_TESTS_SUBDIRS)
-# Only visit subdirs when building, etc because ATF does this it on its own.
-.if !make(atf-test)
-SUBDIR+= ${ATF_TESTS_SUBDIRS}
-.endif
-ATF_TESTS+= ${ATF_TESTS_SUBDIRS}
+# Knob to control the handling of the Atffile for this Makefile.
+#
+# If 'yes', an Atffile exists in the source tree and is installed into
+# TESTSDIR.
+#
+# If 'auto', an Atffile is automatically generated based on the list of test
+# programs built by the Makefile and is installed into TESTSDIR. This is the
+# default and is sufficient in the majority of the cases.
+#
+# If 'no', no Atffile is installed.
+ATFFILE?= auto
-.include <bsd.subdir.mk>
-.endif
+# Path to the prefix of the installed ATF tools, if any.
+#
+# If atf-run and atf-report are installed from ports, we automatically define a
+# realtest target below to run the tests using these tools. The tools are
+# searched for in the hierarchy specified by this variable.
+ATF_PREFIX?= /usr/local
+
+# C compiler passed to ATF tests that need to build code.
+ATF_BUILD_CC?= ${DESTDIR}/usr/bin/cc
+TESTS_ENV+= ATF_BUILD_CC=${ATF_BUILD_CC}
+
+# C preprocessor passed to ATF tests that need to build code.
+ATF_BUILD_CPP?= ${DESTDIR}/usr/bin/cpp
+TESTS_ENV+= ATF_BUILD_CPP=${ATF_BUILD_CPP}
+
+# C++ compiler passed to ATF tests that need to build code.
+ATF_BUILD_CXX?= ${DESTDIR}/usr/bin/c++
+TESTS_ENV+= ATF_BUILD_CXX=${ATF_BUILD_CXX}
-.if defined(TESTS_C)
-ATF_TESTS+= ${TESTS_C}
-.for _T in ${TESTS_C}
+# Shell interpreter used to run atf-sh(1) based tests.
+ATF_SHELL?= ${DESTDIR}/bin/sh
+TESTS_ENV+= ATF_SHELL=${ATF_SHELL}
+
+.if !empty(ATF_TESTS_C)
+PROGS+= ${ATF_TESTS_C}
+_TESTS+= ${ATF_TESTS_C}
+.for _T in ${ATF_TESTS_C}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
SRCS.${_T}?= ${_T}.c
DPADD.${_T}+= ${LIBATF_C}
LDADD.${_T}+= -latf-c
+TEST_INTERFACE.${_T}= atf
.endfor
.endif
-.if defined(TESTS_CXX)
-ATF_TESTS+= ${TESTS_CXX}
-.for _T in ${TESTS_CXX}
+.if !empty(ATF_TESTS_CXX)
+PROGS_CXX+= ${ATF_TESTS_CXX}
+_TESTS+= ${ATF_TESTS_CXX}
+.for _T in ${ATF_TESTS_CXX}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
SRCS.${_T}?= ${_T}${CXX_SUFFIX:U.cc}
DPADD.${_T}+= ${LIBATF_CXX} ${LIBATF_C}
LDADD.${_T}+= -latf-c++ -latf-c
+TEST_INTERFACE.${_T}= atf
.endfor
.endif
-.if defined(TESTS_SH)
-ATF_TESTS+= ${TESTS_SH}
-.for _T in ${TESTS_SH}
+.if !empty(ATF_TESTS_SH)
+SCRIPTS+= ${ATF_TESTS_SH}
+_TESTS+= ${ATF_TESTS_SH}
+.for _T in ${ATF_TESTS_SH}
+SCRIPTSDIR_${_T}= ${TESTSDIR}
+TEST_INTERFACE.${_T}= atf
CLEANFILES+= ${_T} ${_T}.tmp
-TESTS_SH_SRC_${_T}?= ${_T}.sh
-${_T}: ${TESTS_SH_SRC_${_T}}
+ATF_TESTS_SH_SRC_${_T}?= ${_T}.sh
+${_T}: ${ATF_TESTS_SH_SRC_${_T}}
echo '#! /usr/bin/atf-sh' > ${.TARGET}.tmp
cat ${.ALLSRC} >> ${.TARGET}.tmp
chmod +x ${.TARGET}.tmp
@@ -51,4 +101,68 @@ ${_T}: ${TESTS_SH_SRC_${_T}}
.endfor
.endif
+.if ${ALLOW_DEPRECATED_ATF_TOOLS} != "no"
+
+.if ${ATFFILE:tl} != "no"
+FILES+= Atffile
+FILESDIR_Atffile= ${TESTSDIR}
+
+.if ${ATFFILE:tl} == "auto"
+CLEANFILES+= Atffile Atffile.tmp
+
+Atffile: Makefile
+ @{ echo 'Content-Type: application/X-atf-atffile; version="1"'; \
+ echo; \
+ echo '# Automatically generated by atf-test.mk.'; \
+ echo; \
+ echo 'prop: test-suite = "'${TESTSUITE}'"'; \
+ echo; \
+ for tp in ${ATF_TESTS_C} ${ATF_TESTS_CXX} ${ATF_TESTS_SH} \
+ ${TESTS_SUBDIRS}; \
+ do \
+ echo "tp: $${tp}"; \
+ done; } >Atffile.tmp
+ @mv Atffile.tmp Atffile
+.endif
+.endif
+
+ATF_REPORT?= ${ATF_PREFIX}/bin/atf-report
+ATF_RUN?= ${ATF_PREFIX}/bin/atf-run
+.if exists(${ATF_RUN}) && exists(${ATF_REPORT})
+# Definition of the "make test" target and supporting variables.
+#
+# This target, by necessity, can only work for native builds (i.e. a freeBSD
+# host building a release for the same system). The target runs ATF, which is
+# not in the toolchain, and the tests execute code built for the target host.
+#
+# Due to the dependencies of the binaries built by the source tree and how they
+# are used by tests, it is highly possible for a execution of "make test" to
+# report bogus results unless the new binaries are put in place.
+_TESTS_FIFO= ${.OBJDIR}/atf-run.fifo
+_TESTS_LOG= ${.OBJDIR}/atf-run.log
+CLEANFILES+= ${_TESTS_FIFO} ${_TESTS_LOG}
+realtest: .PHONY
+ @set -e; \
+ if [ -z "${TESTSDIR}" ]; then \
+ echo "*** No TESTSDIR defined; nothing to do."; \
+ exit 0; \
+ fi; \
+ cd ${DESTDIR}${TESTSDIR}; \
+ rm -f ${_TESTS_FIFO}; \
+ mkfifo ${_TESTS_FIFO}; \
+ tee ${_TESTS_LOG} < ${_TESTS_FIFO} | ${TESTS_ENV} ${ATF_REPORT} & \
+ set +e; \
+ ${TESTS_ENV} ${ATF_RUN} >> ${_TESTS_FIFO}; \
+ result=$${?}; \
+ wait; \
+ rm -f ${_TESTS_FIFO}; \
+ echo; \
+ echo "*** The verbatim output of atf-run has been saved to ${_TESTS_LOG}"; \
+ echo "***"; \
+ echo "*** WARNING: atf-run is deprecated; please install kyua instead"; \
+ exit $${result}
+.endif
+
+.endif
+
.include <bsd.test.mk>
diff --git a/share/mk/bsd.own.mk b/share/mk/bsd.own.mk
index e2f9894..972a526 100644
--- a/share/mk/bsd.own.mk
+++ b/share/mk/bsd.own.mk
@@ -659,4 +659,8 @@ $xGRP= ${_gid}
.endif # !_WITHOUT_SRCCONF
+# Pointer to the top directory into which tests are installed. Should not be
+# overriden by Makefiles, but the user may choose to set this in src.conf(5).
+TESTSBASE?= /usr/tests
+
.endif # !target(__<bsd.own.mk>__)
diff --git a/share/mk/bsd.test.mk b/share/mk/bsd.test.mk
index a75298f..8cdf914 100644
--- a/share/mk/bsd.test.mk
+++ b/share/mk/bsd.test.mk
@@ -1,33 +1,73 @@
-# $NetBSD: bsd.test.mk,v 1.21 2012/08/25 22:21:16 jmmv Exp $
# $FreeBSD$
+#
+# Generic build infrastructure for test programs.
+#
+# The code in this file is independent of the implementation of the test
+# programs being built; this file just provides generic infrastructure for the
+# build and the definition of various helper variables and targets.
+#
+# Makefiles should never include this file directly. Instead, they should
+# include one of the various *.test.mk depending on the specific test programs
+# being built.
.include <bsd.init.mk>
-.if defined(TESTS_C)
-PROGS+= ${TESTS_C}
-.for _T in ${TESTS_C}
-BINDIR.${_T}= ${TESTSDIR}
-MAN.${_T}?= # empty
-.endfor
-.endif
+# Directory in which to install tests defined by the current Makefile.
+# Makefiles have to override this to point to a subdirectory of TESTSBASE.
+TESTSDIR?= .
-.if defined(TESTS_CXX)
-PROGS_CXX+= ${TESTS_CXX}
-PROGS+= ${TESTS_CXX}
-.for _T in ${TESTS_CXX}
-BINDIR.${_T}= ${TESTSDIR}
-MAN.${_T}?= # empty
-.endfor
-.endif
+# Name of the test suite these tests belong to. Should rarely be changed for
+# Makefiles built into the FreeBSD src tree.
+TESTSUITE?= FreeBSD
-.if defined(TESTS_SH)
-SCRIPTS+= ${TESTS_SH}
-.for _T in ${TESTS_SH}
-SCRIPTSDIR_${_T}= ${TESTSDIR}
-.endfor
-.endif
+# List of subdirectories containing tests into which to recurse. This has the
+# same semantics as SUBDIR at build-time. However, the directories listed here
+# get registered into the run-time test suite definitions so that the test
+# engines know to recurse into these directories.
+#
+# In other words: list here any directories that contain test programs but use
+# SUBDIR for directories that may contain helper binaries and/or data files.
+TESTS_SUBDIRS?=
+
+# Knob to control the handling of the Kyuafile for this Makefile.
+#
+# If 'yes', a Kyuafile exists in the source tree and is installed into
+# TESTSDIR.
+#
+# If 'auto', a Kyuafile is automatically generated based on the list of test
+# programs built by the Makefile and is installed into TESTSDIR. This is the
+# default and is sufficient in the majority of the cases.
+#
+# If 'no', no Kyuafile is installed.
+KYUAFILE?= auto
+
+# List of variables to pass to the tests at run-time via the environment.
+TESTS_ENV?=
+
+# Ordered list of directories to construct the PATH for the tests.
+TESTS_PATH+= ${DESTDIR}/bin ${DESTDIR}/sbin \
+ ${DESTDIR}/usr/bin ${DESTDIR}/usr/sbin
+TESTS_ENV+= PATH=${TESTS_PATH:tW:C/ +/:/g}
-TESTSBASE?= ${DESTDIR}/usr/tests
+# Ordered list of directories to construct the LD_LIBRARY_PATH for the tests.
+TESTS_LD_LIBRARY_PATH+= ${DESTDIR}/lib ${DESTDIR}/usr/lib
+TESTS_ENV+= LD_LIBRARY_PATH=${TESTS_LD_LIBRARY_PATH:tW:C/ +/:/g}
+
+# List of all tests being built. This variable is internal should not be
+# defined by the Makefile. The various *.test.mk modules extend this variable
+# as needed.
+_TESTS?=
+
+# Path to the prefix of the installed Kyua CLI, if any.
+#
+# If kyua is installed from ports, we automatically define a realtest target
+# below to run the tests using this tool. The tools are searched for in the
+# hierarchy specified by this variable.
+KYUA_PREFIX?= /usr/local
+
+.if !empty(TESTS_SUBDIRS)
+SUBDIR+= ${TESTS_SUBDIRS}
+.endif
# it is rare for test cases to have man pages
.if !defined(MAN)
@@ -39,8 +79,57 @@ WITHOUT_MAN=yes
PROG_VARS+= BINDIR
PROGS_TARGETS+= install
-.if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS)
-.include <bsd.progs.mk>
+.if ${KYUAFILE:tl} != "no"
+FILES+= Kyuafile
+FILESDIR_Kyuafile= ${TESTSDIR}
+
+.if ${KYUAFILE:tl} == "auto"
+CLEANFILES+= Kyuafile Kyuafile.tmp
+
+Kyuafile: Makefile
+ @{ \
+ echo '-- Automatically generated by bsd.test.mk.'; \
+ echo; \
+ echo 'syntax(2)'; \
+ echo; \
+ echo 'test_suite("${TESTSUITE}")'; \
+ echo; \
+ } >Kyuafile.tmp
+.for _T in ${_TESTS}
+ @echo "${TEST_INTERFACE.${_T}}_test_program{name=\"${_T}\"}" \
+ >>Kyuafile.tmp
+.endfor
+.for _T in ${TESTS_SUBDIRS:N.WAIT}
+ @echo "include(\"${_T}/Kyuafile\")" >>Kyuafile.tmp
+.endfor
+ @mv Kyuafile.tmp Kyuafile
+.endif
+.endif
+
+KYUA?= ${KYUA_PREFIX}/bin/kyua
+.if exists(${KYUA})
+# Definition of the "make test" target and supporting variables.
+#
+# This target, by necessity, can only work for native builds (i.e. a FreeBSD
+# host building a release for the same system). The target runs Kyua, which is
+# not in the toolchain, and the tests execute code built for the target host.
+#
+# Due to the dependencies of the binaries built by the source tree and how they
+# are used by tests, it is highly possible for a execution of "make test" to
+# report bogus results unless the new binaries are put in place.
+realtest: .PHONY
+ @echo "*** WARNING: make test is experimental"
+ @echo "***"
+ @echo "*** Using this test does not preclude you from running the tests"
+ @echo "*** installed in ${TESTSBASE}. This test run may raise false"
+ @echo "*** positives and/or false negatives."
+ @echo
+ @set -e; \
+ ${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \
+ result=0; \
+ echo; \
+ echo "*** Once again, note that "make test" is unsupported."; \
+ test $${result} -eq 0
.endif
beforetest: .PHONY
@@ -76,4 +165,14 @@ test: beforetest realtest
test: aftertest
.endif
+.if !empty(SUBDIR)
+.include <bsd.subdir.mk>
+.endif
+
+.if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS)
+.include <bsd.progs.mk>
+.elif !empty(FILES)
+.include <bsd.files.mk>
+.endif
+
.include <bsd.obj.mk>
diff --git a/share/mk/plain.test.mk b/share/mk/plain.test.mk
new file mode 100644
index 0000000..b41c585
--- /dev/null
+++ b/share/mk/plain.test.mk
@@ -0,0 +1,52 @@
+# $FreeBSD$
+#
+# Logic to build and install plain test programs. A plain test programs it not
+# supposed to use any specific testing framework: all it does is run some code
+# and report the test's pass or fail status via a 0 or 1 exit code.
+
+.include <bsd.init.mk>
+
+# List of C, C++ and shell test programs to build.
+#
+# Programs listed here are built according to the semantics of bsd.prog.mk for
+# PROGS, PROGS_CXX and SCRIPTS, respectively.
+#
+# Test programs registered in this manner are set to be installed into TESTSDIR
+# (which should be overriden by the Makefile) and are not required to provide a
+# manpage.
+PLAIN_TESTS_C?=
+PLAIN_TESTS_CXX?=
+PLAIN_TESTS_SH?=
+
+.if !empty(PLAIN_TESTS_C)
+PROGS+= ${PLAIN_TESTS_C}
+_TESTS+= ${PLAIN_TESTS_C}
+.for _T in ${PLAIN_TESTS_C}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.c
+TEST_INTERFACE.${_T}= plain
+.endfor
+.endif
+
+.if !empty(PLAIN_TESTS_CXX)
+PROGS_CXX+= ${PLAIN_TESTS_CXX}
+_TESTS+= ${PLAIN_TESTS_CXX}
+.for _T in ${PLAIN_TESTS_CXX}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.cc
+TEST_INTERFACE.${_T}= plain
+.endfor
+.endif
+
+.if !empty(PLAIN_TESTS_SH)
+SCRIPTS+= ${PLAIN_TESTS_SH}
+_TESTS+= ${PLAIN_TESTS_SH}
+.for _T in ${PLAIN_TESTS_SH}
+SCRIPTSDIR_${_T}= ${TESTSDIR}
+TEST_INTERFACE.${_T}= plain
+.endfor
+.endif
+
+.include <bsd.test.mk>
diff --git a/share/mk/tap.test.mk b/share/mk/tap.test.mk
new file mode 100644
index 0000000..394f685
--- /dev/null
+++ b/share/mk/tap.test.mk
@@ -0,0 +1,64 @@
+# $FreeBSD$
+#
+# Logic to build and install TAP-compliant test programs.
+#
+# This is provided to support existing tests in the FreeBSD source tree
+# (particularly those coming from tools/regression/) that comply with the
+# Test Anything Protocol. It should not be used for new tests.
+
+.include <bsd.init.mk>
+
+# List of C, C++ and shell test programs to build.
+#
+# Programs listed here are built according to the semantics of bsd.prog.mk for
+# PROGS, PROGS_CXX and SCRIPTS, respectively.
+#
+# Test programs registered in this manner are set to be installed into TESTSDIR
+# (which should be overriden by the Makefile) and are not required to provide a
+# manpage.
+TAP_TESTS_C?=
+TAP_TESTS_CXX?=
+TAP_TESTS_SH?=
+
+.if !empty(TAP_TESTS_C)
+PROGS+= ${TAP_TESTS_C}
+_TESTS+= ${TAP_TESTS_C}
+.for _T in ${TAP_TESTS_C}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.c
+TEST_INTERFACE.${_T}= tap
+.endfor
+.endif
+
+.if !empty(TAP_TESTS_CXX)
+PROGS_CXX+= ${TAP_TESTS_CXX}
+_TESTS+= ${TAP_TESTS_CXX}
+.for _T in ${TAP_TESTS_CXX}
+BINDIR.${_T}= ${TESTSDIR}
+MAN.${_T}?= # empty
+SRCS.${_T}?= ${_T}.cc
+TEST_INTERFACE.${_T}= tap
+.endfor
+.endif
+
+.if !empty(TAP_TESTS_SH)
+SCRIPTS+= ${TAP_TESTS_SH}
+_TESTS+= ${TAP_TESTS_SH}
+.for _T in ${TAP_TESTS_SH}
+SCRIPTSDIR_${_T}= ${TESTSDIR}
+TEST_INTERFACE.${_T}= tap
+CLEANFILES+= ${_T} ${_T}.tmp
+# TODO(jmmv): It seems to me that this SED and SRC functionality should
+# exist in bsd.prog.mk along the support for SCRIPTS. Move it there if
+# this proves to be useful within the tests.
+TAP_TESTS_SH_SED_${_T}?= # empty
+TAP_TESTS_SH_SRC_${_T}?= ${_T}.sh
+${_T}: ${TAP_TESTS_SH_SRC_${_T}}
+ cat ${.ALLSRC} | sed ${TAP_TESTS_SH_SED_${_T}} >${.TARGET}.tmp
+ chmod +x ${.TARGET}.tmp
+ mv ${.TARGET}.tmp ${.TARGET}
+.endfor
+.endif
+
+.include <bsd.test.mk>
OpenPOWER on IntegriCloud