MFC r269902,r270101:

r269902:

Convert bin/sh/tests to ATF

The new code uses a "test discovery mechanism" to determine
what tests are available for execution

The test shell can be specified via:

  kyua test -v test_suites.FreeBSD.bin.sh.test_shell=/path/to/test/sh

Sponsored by: EMC / Isilon Storage Division
Approved by: jmmv (mentor)
Reviewed by: jilles (maintainer)

r270101 (by jilles):

sh: Don't hardcode relative paths in the tests stderr files.

These paths have had to be adjusted to changes in the testsuite runner
several times, so modify the tests to remove the need for such adjustment.

A cp in functional_test.sh is now unneeded, but this matters little in
performance.

20 files changed, 125 insertions, 86 deletions

diff --git a/bin/sh/tests/Makefile b/bin/sh/tests/Makefile
index 51c6dc4..c092962 100644
--- a/bin/sh/tests/Makefile
+++ b/bin/sh/tests/Makefile
@@ -4,15 +4,12 @@
 
 TESTSDIR=	${TESTSBASE}/bin/sh
 
-TAP_TESTS_SH=	legacy_test
-TAP_TESTS_SH_SED_legacy_test=	-e 's,__SH__,/bin/sh,g'
-# Some tests in here are silently not run when the tests are executed as
-# root.  Explicitly tell Kyua to drop privileges.
-#
-# TODO(jmmv): Kyua needs to do this by default, not only when explicitly
-# requested.  See https://code.google.com/p/kyua/issues/detail?id=6
-TEST_METADATA.legacy_test+= required_user="unprivileged"
-
-SUBDIR+=	builtins errors execution expansion parameters parser set-e
+TESTS_SUBDIRS+=	builtins
+TESTS_SUBDIRS+=	errors
+TESTS_SUBDIRS+=	execution
+TESTS_SUBDIRS+=	expansion
+TESTS_SUBDIRS+=	parameters
+TESTS_SUBDIRS+=	parser
+TESTS_SUBDIRS+=	set-e
 
 .include <bsd.test.mk>
diff --git a/bin/sh/tests/builtins/Makefile b/bin/sh/tests/builtins/Makefile
index ec4cab6..527c1b3 100644
--- a/bin/sh/tests/builtins/Makefile
+++ b/bin/sh/tests/builtins/Makefile
@@ -2,8 +2,12 @@
 
 .include <bsd.own.mk>
 
-FILESDIR=	${TESTSBASE}/bin/sh/builtins
-KYUAFILE=	no
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
+
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		alias.0 alias.0.stdout
 FILES+=		alias.1 alias.1.stderr
diff --git a/bin/sh/tests/errors/Makefile b/bin/sh/tests/errors/Makefile
index ab04f14..51a766f 100644
--- a/bin/sh/tests/errors/Makefile
+++ b/bin/sh/tests/errors/Makefile
@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/errors
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		assignment-error1.0
 FILES+=		assignment-error2.0
diff --git a/bin/sh/tests/errors/bad-parm-exp2.2 b/bin/sh/tests/errors/bad-parm-exp2.2
index 7e13d2b..a0826ec 100644
--- a/bin/sh/tests/errors/bad-parm-exp2.2
+++ b/bin/sh/tests/errors/bad-parm-exp2.2
@@ -1,2 +1,2 @@
 # $FreeBSD$
-${}
+eval '${}'
diff --git a/bin/sh/tests/errors/bad-parm-exp2.2.stderr b/bin/sh/tests/errors/bad-parm-exp2.2.stderr
index d027a5a..51ea69c 100644
--- a/bin/sh/tests/errors/bad-parm-exp2.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp2.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp2.2: ${}: Bad substitution
+eval: ${}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp3.2 b/bin/sh/tests/errors/bad-parm-exp3.2
index a5ecba5..bb41208 100644
--- a/bin/sh/tests/errors/bad-parm-exp3.2
+++ b/bin/sh/tests/errors/bad-parm-exp3.2
@@ -1,2 +1,2 @@
 # $FreeBSD$
-${foo/}
+eval '${foo/}'
diff --git a/bin/sh/tests/errors/bad-parm-exp3.2.stderr b/bin/sh/tests/errors/bad-parm-exp3.2.stderr
index ef40251..70473f9 100644
--- a/bin/sh/tests/errors/bad-parm-exp3.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp3.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp3.2: ${foo/}: Bad substitution
+eval: ${foo/}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp4.2 b/bin/sh/tests/errors/bad-parm-exp4.2
index 9eec8d0..2837f9b 100644
--- a/bin/sh/tests/errors/bad-parm-exp4.2
+++ b/bin/sh/tests/errors/bad-parm-exp4.2
@@ -1,2 +1,2 @@
 # $FreeBSD$
-${foo:@abc}
+eval '${foo:@abc}'
diff --git a/bin/sh/tests/errors/bad-parm-exp4.2.stderr b/bin/sh/tests/errors/bad-parm-exp4.2.stderr
index 89bd80f..3363f51 100644
--- a/bin/sh/tests/errors/bad-parm-exp4.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp4.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp4.2: ${foo:@...}: Bad substitution
+eval: ${foo:@...}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp5.2 b/bin/sh/tests/errors/bad-parm-exp5.2
index 459281f..1ba343b 100644
--- a/bin/sh/tests/errors/bad-parm-exp5.2
+++ b/bin/sh/tests/errors/bad-parm-exp5.2
@@ -1,2 +1,2 @@
 # $FreeBSD$
-${/}
+eval '${/}'
diff --git a/bin/sh/tests/errors/bad-parm-exp5.2.stderr b/bin/sh/tests/errors/bad-parm-exp5.2.stderr
index 89b1997..13763f8 100644
--- a/bin/sh/tests/errors/bad-parm-exp5.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp5.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp5.2: ${/}: Bad substitution
+eval: ${/}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp6.2 b/bin/sh/tests/errors/bad-parm-exp6.2
index ba51442..b53a91b 100644
--- a/bin/sh/tests/errors/bad-parm-exp6.2
+++ b/bin/sh/tests/errors/bad-parm-exp6.2
@@ -1,2 +1,2 @@
 # $FreeBSD$
-${#foo^}
+eval '${#foo^}'
diff --git a/bin/sh/tests/errors/bad-parm-exp6.2.stderr b/bin/sh/tests/errors/bad-parm-exp6.2.stderr
index dbf14b5..cc56f65 100644
--- a/bin/sh/tests/errors/bad-parm-exp6.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp6.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp6.2: ${foo...}: Bad substitution
+eval: ${foo...}: Bad substitution
diff --git a/bin/sh/tests/execution/Makefile b/bin/sh/tests/execution/Makefile
index 302d0d8..2653d5f 100644
--- a/bin/sh/tests/execution/Makefile
+++ b/bin/sh/tests/execution/Makefile
@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/execution
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		bg1.0
 FILES+=		bg2.0
diff --git a/bin/sh/tests/expansion/Makefile b/bin/sh/tests/expansion/Makefile
index e62fea4..d6f3c1a 100644
--- a/bin/sh/tests/expansion/Makefile
+++ b/bin/sh/tests/expansion/Makefile
@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/expansion
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		arith1.0
 FILES+=		arith2.0
diff --git a/bin/sh/tests/functional_test.sh b/bin/sh/tests/functional_test.sh
new file mode 100755
index 0000000..6980538
--- /dev/null
+++ b/bin/sh/tests/functional_test.sh
@@ -0,0 +1,72 @@
+#
+# Copyright 2014 EMC Corp.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# $FreeBSD$
+
+SRCDIR=$(atf_get_srcdir)
+
+check()
+{
+	local tc=${1}; shift
+
+	export SH=$(atf_config_get bin.sh.test_shell /bin/sh)
+
+	local err_file="${SRCDIR}/${tc}.stderr"
+	[ -f "${err_file}" ] && err_flag="-e file:${err_file}"
+	local out_file="${SRCDIR}/${tc}.stdout"
+	[ -f "${out_file}" ] && out_flag="-o file:${out_file}"
+
+	atf_check -s exit:${tc##*.} ${err_flag} ${out_flag} ${SH} "${SRCDIR}/${tc}"
+}
+
+add_testcase()
+{
+	local tc=${1}
+	local tc_escaped word
+
+	case "${tc%.*}" in
+	*-*)
+		local IFS="-"
+		for word in ${tc%.*}; do
+			tc_escaped="${tc_escaped:+${tc_escaped}_}${word}"
+		done
+		;;
+	*)
+		tc_escaped=${tc%.*}
+		;;
+	esac
+
+	atf_test_case ${tc_escaped}
+	eval "${tc_escaped}_body() { check ${tc}; }"
+	atf_add_test_case ${tc_escaped}
+}
+
+atf_init_test_cases()
+{
+	for path in $(find -Es "${SRCDIR}" -regex '.*\.[0-9]+$'); do
+		add_testcase ${path##*/}
+	done
+}
diff --git a/bin/sh/tests/legacy_test.sh b/bin/sh/tests/legacy_test.sh
deleted file mode 100644
index d43f5dd..0000000
--- a/bin/sh/tests/legacy_test.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-# $FreeBSD$
-
-: ${SH:="__SH__"}
-export SH
-
-# TODO(jmmv): The Kyua TAP interface should be passing us the value of
-# "srcdir" as an environment variable, just as it does with the ATF
-# interface in the form of a configuration variable.  For now, just try
-# to guess this.
-: ${TESTS_DATA:=$(dirname ${0})}
-
-COUNTER=1
-
-do_test() {
-	c=${COUNTER}
-	COUNTER=$((COUNTER+1))
-	${SH} $1 > tmp.stdout 2> tmp.stderr
-	if [ $? -ne $2 ]; then
-		echo "not ok ${c} - ${1} # wrong exit status"
-		rm tmp.stdout tmp.stderr
-		return
-	fi
-	sed -I '' -e "s|^${TESTS_DATA}|.|" tmp.stderr
-	for i in stdout stderr; do
-		if [ -f ${1}.${i} ]; then
-			if ! cmp -s tmp.${i} ${1}.${i}; then
-				echo "not ok ${c} - ${1} # wrong output on ${i}"
-				rm tmp.stdout tmp.stderr
-				return
-			fi
-		elif [ -s tmp.${i} ]; then
-			echo "not ok ${c} - ${1} # wrong output on ${i}"
-			rm tmp.stdout tmp.stderr
-			return
-		fi
-	done
-	echo "ok ${c} - ${1}"
-	rm tmp.stdout tmp.stderr
-}
-
-TESTS=$(find -Es ${TESTS_DATA} -regex ".*\.[0-9]+")
-printf "1..%d\n" $(echo ${TESTS} | wc -w)
-
-for i in ${TESTS} ; do
-	do_test ${i} ${i##*.}
-done
diff --git a/bin/sh/tests/parameters/Makefile b/bin/sh/tests/parameters/Makefile
index e31b78f..aa1462f 100644
--- a/bin/sh/tests/parameters/Makefile
+++ b/bin/sh/tests/parameters/Makefile
@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/parameters
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		env1.0
 FILES+=		exitstatus1.0
diff --git a/bin/sh/tests/parser/Makefile b/bin/sh/tests/parser/Makefile
index b769a30..63f4e8f 100644
--- a/bin/sh/tests/parser/Makefile
+++ b/bin/sh/tests/parser/Makefile
@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/parser
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		alias1.0
 FILES+=		alias2.0
diff --git a/bin/sh/tests/set-e/Makefile b/bin/sh/tests/set-e/Makefile
index 55d7917..f733b60 100644
--- a/bin/sh/tests/set-e/Makefile
+++ b/bin/sh/tests/set-e/Makefile
@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/set-e
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		and1.0
 FILES+=		and2.1