Simplify running the FreeBSD test suite

Replace `make regress` (legacy test make target) and `make test` (incomplete
test make target added with the FreeBSD test suite) with make check as it's
consistent with other open source projects.

`make check` defaults to running tests from `.OBJDIR`, but can be overridden
with the `CHECKDIR` variable.

Add `make checkworld` target to simplify running the FreeBSD test suite from
`TESTSBASE` (i.e. the top-level tests directory), similar to buildworld.

Document `make check` and `make checkworld` in build(7).

Other minor changes:

- Rename intermediate file (`Kyuafile.auto`) to `Kyuafile` to simplify
  `make check`.
- Remove terse warnings attached to `beforetest`/`aftertest`.
- Add kyua binary check to check target in suite.test.mk; error out if it's
  not found

The MFC is [partly] contingent on other build related changes being MFCed.

Differential Revision: https://reviews.freebsd.org/D4406
MFC after: 2 months
X-MFC to: stable/10
Relnotes: yes
Reviewed by: bdrewery, Evan Cramer <eccramer@gmail.com>
Sponsored by: EMC / Isilon Storage Division

1 files changed, 16 insertions, 37 deletions

diff --git a/share/mk/suite.test.mk b/share/mk/suite.test.mk
index 2b87d7f..646d2eb 100644
--- a/share/mk/suite.test.mk
+++ b/share/mk/suite.test.mk
@@ -50,15 +50,12 @@ FILES+=	Kyuafile
 FILESDIR_Kyuafile= ${TESTSDIR}
 .endif
 
-.if ${KYUAFILE:tl} == "auto"
-CLEANFILES+= Kyuafile Kyuafile.tmp
-.endif
-
 .for _T in ${_TESTS}
 _TEST_METADATA.${_T}= ${TEST_METADATA} ${TEST_METADATA.${_T}}
 .endfor
 
 .if ${KYUAFILE:tl} == "auto"
+CLEANFILES+= Kyuafile Kyuafile.tmp
 Kyuafile: Makefile
 	@{ \
 	    echo '-- Automatically generated by bsd.test.mk.'; \
@@ -78,9 +75,11 @@ Kyuafile: Makefile
 	@mv ${.TARGET}.tmp ${.TARGET}
 .endif
 
+CHECKDIR?=	${DESTDIR}${TESTSDIR}
+
 KYUA= ${LOCALBASE}/bin/kyua
-.if exists(${KYUA})
-# Definition of the "make test" target and supporting variables.
+
+# Definition of the "make check" target and supporting variables.
 #
 # This target, by necessity, can only work for native builds (i.e. a FreeBSD
 # host building a release for the same system).  The target runs Kyua, which is
@@ -89,35 +88,15 @@ KYUA= ${LOCALBASE}/bin/kyua
 # Due to the dependencies of the binaries built by the source tree and how they
 # are used by tests, it is highly possible for a execution of "make test" to
 # report bogus results unless the new binaries are put in place.
-realtest: .PHONY
-	@echo "*** WARNING: make test is experimental"
-	@echo "***"
-	@echo "*** Using this test does not preclude you from running the tests"
-	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
-	@echo "*** positives and/or false negatives."
-	@echo
-	@${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \
-	result=0; \
-	echo; \
-	echo "*** Once again, note that "make test" is unsupported."; \
-	test $${result} -eq 0
-.endif
 
-beforetest: .PHONY
-.if defined(TESTSDIR)
-.if ${TESTSDIR} == ${TESTSBASE}
-# Forbid running from ${TESTSBASE}.  It can cause false positives/negatives and
-# it does not cover all the tests (e.g. it misses testing software in external).
-	@echo "*** Sorry, you cannot use make test from src/tests.  Install the"
-	@echo "*** tests into their final location and run them from ${TESTSBASE}"
-	@false
-.else
-	@echo "*** Using this test does not preclude you from running the tests"
-	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
-	@echo "*** positives and/or false negatives."
-.endif
-.else
-	@echo "*** No TESTSDIR defined; nothing to do."
-	@false
-.endif
-	@echo
+realcheck: .PHONY
+	@if [ ! -x ${KYUA} ]; then \
+		echo; \
+		echo "kyua binary not installed at expected location (${.TARGET})"; \
+		echo; \
+		echo "Please install via pkg install, or specify the path to the kyua"; \
+		echo "package via the \$${LOCALBASE} variable, e.g. "; \
+		echo "LOCALBASE=\"${LOCALBASE}\""; \
+		false; \
+	fi
+	@${KYUA} test -k ${CHECKDIR}/Kyuafile