diff options
author | jmmv <jmmv@FreeBSD.org> | 2013-12-27 22:00:22 +0000 |
---|---|---|
committer | jmmv <jmmv@FreeBSD.org> | 2013-12-27 22:00:22 +0000 |
commit | e125f95c9c257e17bc2724cf2f7bdb2a5e1f367c (patch) | |
tree | 1119ed400ee859c1a442dea4edd3494fbc8a2f21 /share | |
parent | 33f454b5c1f0354b8354b4536b45d3d1d1cc8982 (diff) | |
download | FreeBSD-src-e125f95c9c257e17bc2724cf2f7bdb2a5e1f367c.zip FreeBSD-src-e125f95c9c257e17bc2724cf2f7bdb2a5e1f367c.tar.gz |
Split and extend bsd.test.mk into {atf,plain,tap}.test.mk.
This is a MFC of:
- r256761 Clearly split the logic to build ATF and plain tests apart.
- r256762 Add the automatic generation of Atffile files.
- r256763 Add the automatic generation of Kyuafile files.
- r256764 Plug atf-run into the 'test' target.
- r256765 Plug kyua into the 'test' target.
- r257096 Move the TESTSBASE definition to bsd.own.mk.
- r257099 Add missing plain.test.mk.
- r258297 Remove registration of C++ test programs into PROGS.
- r258298 Fix the build of plain test programs.
- r258551 Install plain.test.mk.
- r259208 Add tap.test.mk.
Approved by: rpaulo (mentor)
Diffstat (limited to 'share')
-rw-r--r-- | share/mk/Makefile | 2 | ||||
-rw-r--r-- | share/mk/atf.test.mk | 162 | ||||
-rw-r--r-- | share/mk/bsd.own.mk | 4 | ||||
-rw-r--r-- | share/mk/bsd.test.mk | 149 | ||||
-rw-r--r-- | share/mk/plain.test.mk | 52 | ||||
-rw-r--r-- | share/mk/tap.test.mk | 64 |
6 files changed, 384 insertions, 49 deletions
diff --git a/share/mk/Makefile b/share/mk/Makefile index 2110b26..d07402b 100644 --- a/share/mk/Makefile +++ b/share/mk/Makefile @@ -45,6 +45,8 @@ FILESDIR= ${BINDIR}/mk .if ${MK_ATF} != "no" FILES+= atf.test.mk +FILES+= plain.test.mk +FILES+= tap.test.mk .endif .include <bsd.prog.mk> diff --git a/share/mk/atf.test.mk b/share/mk/atf.test.mk index da4d12f..3ffb2cb 100644 --- a/share/mk/atf.test.mk +++ b/share/mk/atf.test.mk @@ -1,49 +1,99 @@ -# $NetBSD$ # $FreeBSD$ # +# Logic to build and install ATF test programs; i.e. test programs linked +# against the ATF libraries. .include <bsd.init.mk> -ATF_TESTS:= +# List of C, C++ and shell test programs to build. +# +# Programs listed here are built using PROGS, PROGS_CXX and SCRIPTS, +# respectively, from bsd.prog.mk. However, the build rules are tweaked to +# require the ATF libraries. +# +# Test programs registered in this manner are set to be installed into TESTSDIR +# (which should be overriden by the Makefile) and are not required to provide a +# manpage. +ATF_TESTS_C?= +ATF_TESTS_CXX?= +ATF_TESTS_SH?= -.if make(*test) -TESTSDIR?= . -.endif +# Whether to allow using the deprecated ATF tools or not. +# +# If 'yes', this file will generate Atffiles when requested and will also +# support using the deprecated atf-run tool to execute the tests. +ALLOW_DEPRECATED_ATF_TOOLS?= no -.if defined(ATF_TESTS_SUBDIRS) -# Only visit subdirs when building, etc because ATF does this it on its own. -.if !make(atf-test) -SUBDIR+= ${ATF_TESTS_SUBDIRS} -.endif -ATF_TESTS+= ${ATF_TESTS_SUBDIRS} +# Knob to control the handling of the Atffile for this Makefile. +# +# If 'yes', an Atffile exists in the source tree and is installed into +# TESTSDIR. +# +# If 'auto', an Atffile is automatically generated based on the list of test +# programs built by the Makefile and is installed into TESTSDIR. This is the +# default and is sufficient in the majority of the cases. +# +# If 'no', no Atffile is installed. +ATFFILE?= auto -.include <bsd.subdir.mk> -.endif +# Path to the prefix of the installed ATF tools, if any. +# +# If atf-run and atf-report are installed from ports, we automatically define a +# realtest target below to run the tests using these tools. The tools are +# searched for in the hierarchy specified by this variable. +ATF_PREFIX?= /usr/local + +# C compiler passed to ATF tests that need to build code. +ATF_BUILD_CC?= ${DESTDIR}/usr/bin/cc +TESTS_ENV+= ATF_BUILD_CC=${ATF_BUILD_CC} + +# C preprocessor passed to ATF tests that need to build code. +ATF_BUILD_CPP?= ${DESTDIR}/usr/bin/cpp +TESTS_ENV+= ATF_BUILD_CPP=${ATF_BUILD_CPP} + +# C++ compiler passed to ATF tests that need to build code. +ATF_BUILD_CXX?= ${DESTDIR}/usr/bin/c++ +TESTS_ENV+= ATF_BUILD_CXX=${ATF_BUILD_CXX} -.if defined(TESTS_C) -ATF_TESTS+= ${TESTS_C} -.for _T in ${TESTS_C} +# Shell interpreter used to run atf-sh(1) based tests. +ATF_SHELL?= ${DESTDIR}/bin/sh +TESTS_ENV+= ATF_SHELL=${ATF_SHELL} + +.if !empty(ATF_TESTS_C) +PROGS+= ${ATF_TESTS_C} +_TESTS+= ${ATF_TESTS_C} +.for _T in ${ATF_TESTS_C} +BINDIR.${_T}= ${TESTSDIR} +MAN.${_T}?= # empty SRCS.${_T}?= ${_T}.c DPADD.${_T}+= ${LIBATF_C} LDADD.${_T}+= -latf-c +TEST_INTERFACE.${_T}= atf .endfor .endif -.if defined(TESTS_CXX) -ATF_TESTS+= ${TESTS_CXX} -.for _T in ${TESTS_CXX} +.if !empty(ATF_TESTS_CXX) +PROGS_CXX+= ${ATF_TESTS_CXX} +_TESTS+= ${ATF_TESTS_CXX} +.for _T in ${ATF_TESTS_CXX} +BINDIR.${_T}= ${TESTSDIR} +MAN.${_T}?= # empty SRCS.${_T}?= ${_T}${CXX_SUFFIX:U.cc} DPADD.${_T}+= ${LIBATF_CXX} ${LIBATF_C} LDADD.${_T}+= -latf-c++ -latf-c +TEST_INTERFACE.${_T}= atf .endfor .endif -.if defined(TESTS_SH) -ATF_TESTS+= ${TESTS_SH} -.for _T in ${TESTS_SH} +.if !empty(ATF_TESTS_SH) +SCRIPTS+= ${ATF_TESTS_SH} +_TESTS+= ${ATF_TESTS_SH} +.for _T in ${ATF_TESTS_SH} +SCRIPTSDIR_${_T}= ${TESTSDIR} +TEST_INTERFACE.${_T}= atf CLEANFILES+= ${_T} ${_T}.tmp -TESTS_SH_SRC_${_T}?= ${_T}.sh -${_T}: ${TESTS_SH_SRC_${_T}} +ATF_TESTS_SH_SRC_${_T}?= ${_T}.sh +${_T}: ${ATF_TESTS_SH_SRC_${_T}} echo '#! /usr/bin/atf-sh' > ${.TARGET}.tmp cat ${.ALLSRC} >> ${.TARGET}.tmp chmod +x ${.TARGET}.tmp @@ -51,4 +101,68 @@ ${_T}: ${TESTS_SH_SRC_${_T}} .endfor .endif +.if ${ALLOW_DEPRECATED_ATF_TOOLS} != "no" + +.if ${ATFFILE:tl} != "no" +FILES+= Atffile +FILESDIR_Atffile= ${TESTSDIR} + +.if ${ATFFILE:tl} == "auto" +CLEANFILES+= Atffile Atffile.tmp + +Atffile: Makefile + @{ echo 'Content-Type: application/X-atf-atffile; version="1"'; \ + echo; \ + echo '# Automatically generated by atf-test.mk.'; \ + echo; \ + echo 'prop: test-suite = "'${TESTSUITE}'"'; \ + echo; \ + for tp in ${ATF_TESTS_C} ${ATF_TESTS_CXX} ${ATF_TESTS_SH} \ + ${TESTS_SUBDIRS}; \ + do \ + echo "tp: $${tp}"; \ + done; } >Atffile.tmp + @mv Atffile.tmp Atffile +.endif +.endif + +ATF_REPORT?= ${ATF_PREFIX}/bin/atf-report +ATF_RUN?= ${ATF_PREFIX}/bin/atf-run +.if exists(${ATF_RUN}) && exists(${ATF_REPORT}) +# Definition of the "make test" target and supporting variables. +# +# This target, by necessity, can only work for native builds (i.e. a freeBSD +# host building a release for the same system). The target runs ATF, which is +# not in the toolchain, and the tests execute code built for the target host. +# +# Due to the dependencies of the binaries built by the source tree and how they +# are used by tests, it is highly possible for a execution of "make test" to +# report bogus results unless the new binaries are put in place. +_TESTS_FIFO= ${.OBJDIR}/atf-run.fifo +_TESTS_LOG= ${.OBJDIR}/atf-run.log +CLEANFILES+= ${_TESTS_FIFO} ${_TESTS_LOG} +realtest: .PHONY + @set -e; \ + if [ -z "${TESTSDIR}" ]; then \ + echo "*** No TESTSDIR defined; nothing to do."; \ + exit 0; \ + fi; \ + cd ${DESTDIR}${TESTSDIR}; \ + rm -f ${_TESTS_FIFO}; \ + mkfifo ${_TESTS_FIFO}; \ + tee ${_TESTS_LOG} < ${_TESTS_FIFO} | ${TESTS_ENV} ${ATF_REPORT} & \ + set +e; \ + ${TESTS_ENV} ${ATF_RUN} >> ${_TESTS_FIFO}; \ + result=$${?}; \ + wait; \ + rm -f ${_TESTS_FIFO}; \ + echo; \ + echo "*** The verbatim output of atf-run has been saved to ${_TESTS_LOG}"; \ + echo "***"; \ + echo "*** WARNING: atf-run is deprecated; please install kyua instead"; \ + exit $${result} +.endif + +.endif + .include <bsd.test.mk> diff --git a/share/mk/bsd.own.mk b/share/mk/bsd.own.mk index e2f9894..972a526 100644 --- a/share/mk/bsd.own.mk +++ b/share/mk/bsd.own.mk @@ -659,4 +659,8 @@ $xGRP= ${_gid} .endif # !_WITHOUT_SRCCONF +# Pointer to the top directory into which tests are installed. Should not be +# overriden by Makefiles, but the user may choose to set this in src.conf(5). +TESTSBASE?= /usr/tests + .endif # !target(__<bsd.own.mk>__) diff --git a/share/mk/bsd.test.mk b/share/mk/bsd.test.mk index a75298f..8cdf914 100644 --- a/share/mk/bsd.test.mk +++ b/share/mk/bsd.test.mk @@ -1,33 +1,73 @@ -# $NetBSD: bsd.test.mk,v 1.21 2012/08/25 22:21:16 jmmv Exp $ # $FreeBSD$ +# +# Generic build infrastructure for test programs. +# +# The code in this file is independent of the implementation of the test +# programs being built; this file just provides generic infrastructure for the +# build and the definition of various helper variables and targets. +# +# Makefiles should never include this file directly. Instead, they should +# include one of the various *.test.mk depending on the specific test programs +# being built. .include <bsd.init.mk> -.if defined(TESTS_C) -PROGS+= ${TESTS_C} -.for _T in ${TESTS_C} -BINDIR.${_T}= ${TESTSDIR} -MAN.${_T}?= # empty -.endfor -.endif +# Directory in which to install tests defined by the current Makefile. +# Makefiles have to override this to point to a subdirectory of TESTSBASE. +TESTSDIR?= . -.if defined(TESTS_CXX) -PROGS_CXX+= ${TESTS_CXX} -PROGS+= ${TESTS_CXX} -.for _T in ${TESTS_CXX} -BINDIR.${_T}= ${TESTSDIR} -MAN.${_T}?= # empty -.endfor -.endif +# Name of the test suite these tests belong to. Should rarely be changed for +# Makefiles built into the FreeBSD src tree. +TESTSUITE?= FreeBSD -.if defined(TESTS_SH) -SCRIPTS+= ${TESTS_SH} -.for _T in ${TESTS_SH} -SCRIPTSDIR_${_T}= ${TESTSDIR} -.endfor -.endif +# List of subdirectories containing tests into which to recurse. This has the +# same semantics as SUBDIR at build-time. However, the directories listed here +# get registered into the run-time test suite definitions so that the test +# engines know to recurse into these directories. +# +# In other words: list here any directories that contain test programs but use +# SUBDIR for directories that may contain helper binaries and/or data files. +TESTS_SUBDIRS?= + +# Knob to control the handling of the Kyuafile for this Makefile. +# +# If 'yes', a Kyuafile exists in the source tree and is installed into +# TESTSDIR. +# +# If 'auto', a Kyuafile is automatically generated based on the list of test +# programs built by the Makefile and is installed into TESTSDIR. This is the +# default and is sufficient in the majority of the cases. +# +# If 'no', no Kyuafile is installed. +KYUAFILE?= auto + +# List of variables to pass to the tests at run-time via the environment. +TESTS_ENV?= + +# Ordered list of directories to construct the PATH for the tests. +TESTS_PATH+= ${DESTDIR}/bin ${DESTDIR}/sbin \ + ${DESTDIR}/usr/bin ${DESTDIR}/usr/sbin +TESTS_ENV+= PATH=${TESTS_PATH:tW:C/ +/:/g} -TESTSBASE?= ${DESTDIR}/usr/tests +# Ordered list of directories to construct the LD_LIBRARY_PATH for the tests. +TESTS_LD_LIBRARY_PATH+= ${DESTDIR}/lib ${DESTDIR}/usr/lib +TESTS_ENV+= LD_LIBRARY_PATH=${TESTS_LD_LIBRARY_PATH:tW:C/ +/:/g} + +# List of all tests being built. This variable is internal should not be +# defined by the Makefile. The various *.test.mk modules extend this variable +# as needed. +_TESTS?= + +# Path to the prefix of the installed Kyua CLI, if any. +# +# If kyua is installed from ports, we automatically define a realtest target +# below to run the tests using this tool. The tools are searched for in the +# hierarchy specified by this variable. +KYUA_PREFIX?= /usr/local + +.if !empty(TESTS_SUBDIRS) +SUBDIR+= ${TESTS_SUBDIRS} +.endif # it is rare for test cases to have man pages .if !defined(MAN) @@ -39,8 +79,57 @@ WITHOUT_MAN=yes PROG_VARS+= BINDIR PROGS_TARGETS+= install -.if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS) -.include <bsd.progs.mk> +.if ${KYUAFILE:tl} != "no" +FILES+= Kyuafile +FILESDIR_Kyuafile= ${TESTSDIR} + +.if ${KYUAFILE:tl} == "auto" +CLEANFILES+= Kyuafile Kyuafile.tmp + +Kyuafile: Makefile + @{ \ + echo '-- Automatically generated by bsd.test.mk.'; \ + echo; \ + echo 'syntax(2)'; \ + echo; \ + echo 'test_suite("${TESTSUITE}")'; \ + echo; \ + } >Kyuafile.tmp +.for _T in ${_TESTS} + @echo "${TEST_INTERFACE.${_T}}_test_program{name=\"${_T}\"}" \ + >>Kyuafile.tmp +.endfor +.for _T in ${TESTS_SUBDIRS:N.WAIT} + @echo "include(\"${_T}/Kyuafile\")" >>Kyuafile.tmp +.endfor + @mv Kyuafile.tmp Kyuafile +.endif +.endif + +KYUA?= ${KYUA_PREFIX}/bin/kyua +.if exists(${KYUA}) +# Definition of the "make test" target and supporting variables. +# +# This target, by necessity, can only work for native builds (i.e. a FreeBSD +# host building a release for the same system). The target runs Kyua, which is +# not in the toolchain, and the tests execute code built for the target host. +# +# Due to the dependencies of the binaries built by the source tree and how they +# are used by tests, it is highly possible for a execution of "make test" to +# report bogus results unless the new binaries are put in place. +realtest: .PHONY + @echo "*** WARNING: make test is experimental" + @echo "***" + @echo "*** Using this test does not preclude you from running the tests" + @echo "*** installed in ${TESTSBASE}. This test run may raise false" + @echo "*** positives and/or false negatives." + @echo + @set -e; \ + ${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \ + result=0; \ + echo; \ + echo "*** Once again, note that "make test" is unsupported."; \ + test $${result} -eq 0 .endif beforetest: .PHONY @@ -76,4 +165,14 @@ test: beforetest realtest test: aftertest .endif +.if !empty(SUBDIR) +.include <bsd.subdir.mk> +.endif + +.if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS) +.include <bsd.progs.mk> +.elif !empty(FILES) +.include <bsd.files.mk> +.endif + .include <bsd.obj.mk> diff --git a/share/mk/plain.test.mk b/share/mk/plain.test.mk new file mode 100644 index 0000000..b41c585 --- /dev/null +++ b/share/mk/plain.test.mk @@ -0,0 +1,52 @@ +# $FreeBSD$ +# +# Logic to build and install plain test programs. A plain test programs it not +# supposed to use any specific testing framework: all it does is run some code +# and report the test's pass or fail status via a 0 or 1 exit code. + +.include <bsd.init.mk> + +# List of C, C++ and shell test programs to build. +# +# Programs listed here are built according to the semantics of bsd.prog.mk for +# PROGS, PROGS_CXX and SCRIPTS, respectively. +# +# Test programs registered in this manner are set to be installed into TESTSDIR +# (which should be overriden by the Makefile) and are not required to provide a +# manpage. +PLAIN_TESTS_C?= +PLAIN_TESTS_CXX?= +PLAIN_TESTS_SH?= + +.if !empty(PLAIN_TESTS_C) +PROGS+= ${PLAIN_TESTS_C} +_TESTS+= ${PLAIN_TESTS_C} +.for _T in ${PLAIN_TESTS_C} +BINDIR.${_T}= ${TESTSDIR} +MAN.${_T}?= # empty +SRCS.${_T}?= ${_T}.c +TEST_INTERFACE.${_T}= plain +.endfor +.endif + +.if !empty(PLAIN_TESTS_CXX) +PROGS_CXX+= ${PLAIN_TESTS_CXX} +_TESTS+= ${PLAIN_TESTS_CXX} +.for _T in ${PLAIN_TESTS_CXX} +BINDIR.${_T}= ${TESTSDIR} +MAN.${_T}?= # empty +SRCS.${_T}?= ${_T}.cc +TEST_INTERFACE.${_T}= plain +.endfor +.endif + +.if !empty(PLAIN_TESTS_SH) +SCRIPTS+= ${PLAIN_TESTS_SH} +_TESTS+= ${PLAIN_TESTS_SH} +.for _T in ${PLAIN_TESTS_SH} +SCRIPTSDIR_${_T}= ${TESTSDIR} +TEST_INTERFACE.${_T}= plain +.endfor +.endif + +.include <bsd.test.mk> diff --git a/share/mk/tap.test.mk b/share/mk/tap.test.mk new file mode 100644 index 0000000..394f685 --- /dev/null +++ b/share/mk/tap.test.mk @@ -0,0 +1,64 @@ +# $FreeBSD$ +# +# Logic to build and install TAP-compliant test programs. +# +# This is provided to support existing tests in the FreeBSD source tree +# (particularly those coming from tools/regression/) that comply with the +# Test Anything Protocol. It should not be used for new tests. + +.include <bsd.init.mk> + +# List of C, C++ and shell test programs to build. +# +# Programs listed here are built according to the semantics of bsd.prog.mk for +# PROGS, PROGS_CXX and SCRIPTS, respectively. +# +# Test programs registered in this manner are set to be installed into TESTSDIR +# (which should be overriden by the Makefile) and are not required to provide a +# manpage. +TAP_TESTS_C?= +TAP_TESTS_CXX?= +TAP_TESTS_SH?= + +.if !empty(TAP_TESTS_C) +PROGS+= ${TAP_TESTS_C} +_TESTS+= ${TAP_TESTS_C} +.for _T in ${TAP_TESTS_C} +BINDIR.${_T}= ${TESTSDIR} +MAN.${_T}?= # empty +SRCS.${_T}?= ${_T}.c +TEST_INTERFACE.${_T}= tap +.endfor +.endif + +.if !empty(TAP_TESTS_CXX) +PROGS_CXX+= ${TAP_TESTS_CXX} +_TESTS+= ${TAP_TESTS_CXX} +.for _T in ${TAP_TESTS_CXX} +BINDIR.${_T}= ${TESTSDIR} +MAN.${_T}?= # empty +SRCS.${_T}?= ${_T}.cc +TEST_INTERFACE.${_T}= tap +.endfor +.endif + +.if !empty(TAP_TESTS_SH) +SCRIPTS+= ${TAP_TESTS_SH} +_TESTS+= ${TAP_TESTS_SH} +.for _T in ${TAP_TESTS_SH} +SCRIPTSDIR_${_T}= ${TESTSDIR} +TEST_INTERFACE.${_T}= tap +CLEANFILES+= ${_T} ${_T}.tmp +# TODO(jmmv): It seems to me that this SED and SRC functionality should +# exist in bsd.prog.mk along the support for SCRIPTS. Move it there if +# this proves to be useful within the tests. +TAP_TESTS_SH_SED_${_T}?= # empty +TAP_TESTS_SH_SRC_${_T}?= ${_T}.sh +${_T}: ${TAP_TESTS_SH_SRC_${_T}} + cat ${.ALLSRC} | sed ${TAP_TESTS_SH_SED_${_T}} >${.TARGET}.tmp + chmod +x ${.TARGET}.tmp + mv ${.TARGET}.tmp ${.TARGET} +.endfor +.endif + +.include <bsd.test.mk> |