Switch over to a different, more flexible test output protocol that's

understood by Perl's Test::Harness module and prove(1) commands.

Update README to describe the new protocol.  The work's broken down into
two main sets of changes.

First, update the existing test programs (shell scripts and C programs)
to produce output in the ok/not ok format, and to, where possible, also
produce a header describing the number of tests that are expected to be
run.

Second, provide the .t files that actually run the tests.  In some cases
these are copies of, or very similar too, scripts that already existed.
I've kept the old scripts around so that it's possible to verify that
behaviour under this new system (in terms of whether or not a test fails)
is identical to the behaviour under the old system.

Add a TODO file.

6 files changed, 112 insertions, 27 deletions

diff --git a/tools/regression/sockets/accept_fd_leak/accept_fd_leak.c b/tools/regression/sockets/accept_fd_leak/accept_fd_leak.c
index c81230f..9bf946d 100644
--- a/tools/regression/sockets/accept_fd_leak/accept_fd_leak.c
+++ b/tools/regression/sockets/accept_fd_leak/accept_fd_leak.c
@@ -55,6 +55,8 @@ main(int argc, char *argv[])
 	socklen_t size;
 	int fd1, fd2, fd3, i, s;
 
+	printf("1..1\n");
+
 	/*
 	 * Check for sequential fd allocation, and give up early if not.
 	 */
@@ -105,9 +107,9 @@ main(int argc, char *argv[])
 	 */
 	fd3 = dup(STDIN_FILENO);
 	if (fd3 != fd2 + 2)
-		errx(-1, "FAIL (%d, %d, %d)\n", fd1, fd2, fd3);
+		printf("not ok 1 - (%d, %d, %d)\n", fd1, fd2, fd3);
 	else
-		fprintf(stderr, "PASS\n");
+		printf("ok 1\n");
 
 	return (0);
 }
diff --git a/tools/regression/sockets/accept_fd_leak/accept_fd_leak.t b/tools/regression/sockets/accept_fd_leak/accept_fd_leak.t
new file mode 100644
index 0000000..8bdfd03
--- /dev/null
+++ b/tools/regression/sockets/accept_fd_leak/accept_fd_leak.t
@@ -0,0 +1,10 @@
+#!/bin/sh
+# $FreeBSD$
+
+cd `dirname $0`
+
+executable=`basename $0 .t`
+
+make $executable 2>&1 > /dev/null
+
+exec ./$executable
diff --git a/tools/regression/sockets/accf_data_attach/accf_data_attach.c b/tools/regression/sockets/accf_data_attach/accf_data_attach.c
index 06809d7..ccf8d55 100644
--- a/tools/regression/sockets/accf_data_attach/accf_data_attach.c
+++ b/tools/regression/sockets/accf_data_attach/accf_data_attach.c
@@ -63,12 +63,15 @@ main(int argc, char *argv[])
 	socklen_t len;
 	int lso, ret;
 
+	printf("1..9\n");
+
 	/*
 	 * Step 0. Open socket().
 	 */
 	lso = socket(PF_INET, SOCK_STREAM, 0);
 	if (lso == -1)
-		errx(-1, "socket: %s", strerror(errno));
+		errx(-1, "not ok 1 - socket: %s", strerror(errno));
+	printf("ok 1 - socket\n");
 
 	/*
 	 * Step 1. After socket().  Should return EINVAL, since no accept
@@ -78,10 +81,11 @@ main(int argc, char *argv[])
 	len = sizeof(afa);
 	ret = getsockopt(lso, SOL_SOCKET, SO_ACCEPTFILTER, &afa, &len);
 	if (ret != -1)
-		errx(-1, "FAIL: getsockopt() after socket() succeeded");
+		errx(-1, "not ok 2 - getsockopt() after socket() succeeded");
 	if (errno != EINVAL)
-		errx(-1, "FAIL: getsockopt() after socket() failed with "
+		errx(-1, "not ok 2 - getsockopt() after socket() failed with "
 		    "%d (%s)", errno, strerror(errno));
+	printf("ok 2 - getsockopt\n");
 
 	/*
 	 * Step 2. Bind().  Ideally this will succeed.
@@ -92,7 +96,8 @@ main(int argc, char *argv[])
 	sin.sin_port = htons(8080);
 	sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 	if (bind(lso, (struct sockaddr *)&sin, sizeof(sin)) < 0)
-		errx(-1, "bind: %s", strerror(errno));
+		errx(-1, "not ok 3 - bind %s", strerror(errno));
+	printf("ok 3 - bind\n");
 
 	/*
 	 * Step 3: After bind().  getsockopt() should return EINVAL, since no
@@ -101,10 +106,11 @@ main(int argc, char *argv[])
 	len = sizeof(afa);
 	ret = getsockopt(lso, SOL_SOCKET, SO_ACCEPTFILTER, &afa, &len);
 	if (ret != -1)
-		errx(-1, "FAIL: getsockopt() after bind() succeeded");
+		errx(-1, "not ok 4 - getsockopt() after bind() succeeded");
 	if (errno != EINVAL)
-		errx(-1, "FAIL: getsockopt() after bind() failed with %d (%s)",
+		errx(-1, "not ok 4 -  getsockopt() after bind() failed with %d (%s)",
 		    errno, strerror(errno));
+	printf("ok 4 - getsockopt\n");
 
 	/*
 	 * Step 4: Setsockopt() before listen().  Should fail, since it's not
@@ -114,7 +120,8 @@ main(int argc, char *argv[])
 	strcpy(afa.af_name, ACCF_NAME);
 	ret = setsockopt(lso, SOL_SOCKET, SO_ACCEPTFILTER, &afa, sizeof(afa));
 	if (ret == 0)
-		errx(-1, "FAIL: setsockopt() before listen() succeeded");
+		errx(-1, "not ok 5 - setsockopt() before listen() succeeded");
+	printf("ok 5 - setsockopt\n");
 
 	/*
 	 * Step 5: Getsockopt() after pre-listen() setsockopt().  Should
@@ -123,17 +130,19 @@ main(int argc, char *argv[])
 	len = sizeof(afa);
 	ret = getsockopt(lso, SOL_SOCKET, SO_ACCEPTFILTER, &afa, &len);
 	if (ret == 0)
-		errx(-1, "FAIL: getsockopt() after pre-listen() setsockopt() "
+		errx(-1, "not ok 6 - getsockopt() after pre-listen() setsockopt() "
 		    "succeeded");
 	if (errno != EINVAL)
-		errx(-1, "FAIL: pre-listen() getsockopt() failed with %d (%s)",
+		errx(-1, "not ok 6 - pre-listen() getsockopt() failed with %d (%s)",
 		    errno, strerror(errno));
+	printf("ok 6 - getsockopt\n");
 
 	/*
 	 * Step 6: listen().
 	 */
 	if (listen(lso, -1) < 0)
-		errx(-1, "listen: %s", strerror(errno));
+		errx(-1, "not ok 7 - listen: %s", strerror(errno));
+	printf("ok 7 - listen\n");
 
 	/*
 	 * Step 7: After listen().  This call to setsockopt() should succeed.
@@ -142,11 +151,12 @@ main(int argc, char *argv[])
 	strcpy(afa.af_name, ACCF_NAME);
 	ret = setsockopt(lso, SOL_SOCKET, SO_ACCEPTFILTER, &afa, sizeof(afa));
 	if (ret != 0)
-		errx(-1, "FAIL: setsockopt() after listen() failed with %d "
+		errx(-1, "not ok 8 - setsockopt() after listen() failed with %d "
 		    "(%s)", errno, strerror(errno));
 	if (len != sizeof(afa))
-		errx(-1, "FAIL: setsockopt() after listen() returned wrong "
+		errx(-1, "not ok 8 - setsockopt() after listen() returned wrong "
 		    "size (%d vs expected %d)", len, sizeof(afa));
+	printf("ok 8 - setsockopt\n");
 
 	/*
 	 * Step 8: After setsockopt().  Should succeed and identify
@@ -156,18 +166,18 @@ main(int argc, char *argv[])
 	len = sizeof(afa);
 	ret = getsockopt(lso, SOL_SOCKET, SO_ACCEPTFILTER, &afa, &len);
 	if (ret != 0)
-		errx(-1, "FAIL: getsockopt() after listen() setsockopt() "
+		errx(-1, "not ok 9 - getsockopt() after listen() setsockopt() "
 		    "failed with %d (%s)", errno, strerror(errno));
 	if (len != sizeof(afa))
-		errx(-1, "FAIL: getsockopt() after setsockopet()  after "
+		errx(-1, "not ok 9 - getsockopt() after setsockopet()  after "
 		    "listen() returned wrong size (got %d expected %d)", len,
 		    sizeof(afa));
 	if (strcmp(afa.af_name, ACCF_NAME) != 0)
-		errx(-1, "FAIL: getsockopt() after setsockopt() after "
+		errx(-1, "not ok 9 - getsockopt() after setsockopt() after "
 		    "listen() mismatch (got %s expected %s)", afa.af_name,
 		    ACCF_NAME);
+	printf("ok 9 - getsockopt\n");
 
-	printf("PASS\n");
 	close(lso);
 	return (0);
 }
diff --git a/tools/regression/sockets/accf_data_attach/accf_data_attach.t b/tools/regression/sockets/accf_data_attach/accf_data_attach.t
new file mode 100644
index 0000000..8bdfd03
--- /dev/null
+++ b/tools/regression/sockets/accf_data_attach/accf_data_attach.t
@@ -0,0 +1,10 @@
+#!/bin/sh
+# $FreeBSD$
+
+cd `dirname $0`
+
+executable=`basename $0 .t`
+
+make $executable 2>&1 > /dev/null
+
+exec ./$executable
diff --git a/tools/regression/sockets/kqueue/kqueue.c b/tools/regression/sockets/kqueue/kqueue.c
index 4c94ec0..d46db96 100644
--- a/tools/regression/sockets/kqueue/kqueue.c
+++ b/tools/regression/sockets/kqueue/kqueue.c
@@ -38,6 +38,8 @@
 #include <string.h>
 #include <unistd.h>
 
+static int	curtest = 1;
+
 /*-
  * This test uses UNIX domain socket pairs to perform some basic exercising
  * of kqueue functionality on sockets.  In particular, testing that for read
@@ -53,19 +55,22 @@
  *   closed.
  */
 
+#define OK(testname)	printf("ok %d - %s\n", curtest, testname); \
+			curtest++;
+
 static void
 fail(int error, const char *func, const char *socktype, const char *rest)
 {
 
-	fprintf(stderr, "FAIL\n");
+	printf("not ok %d\n", curtest);
 
 	if (socktype == NULL)
-		fprintf(stderr, "%s(): %s\n", func, strerror(error));
+		printf("# %s(): %s\n", func, strerror(error));
 	else if (rest == NULL)
-		fprintf(stderr, "%s(%s): %s\n", func, socktype,
+		printf("# %s(%s): %s\n", func, socktype,
 		    strerror(error));
 	else
-		fprintf(stderr, "%s(%s, %s): %s\n", func, socktype, rest,
+		printf("# %s(%s, %s): %s\n", func, socktype, rest,
 		    strerror(error));
 	exit(-1);
 }
@@ -75,16 +80,16 @@ fail_assertion(const char *func, const char *socktype, const char *rest,
     const char *assertion)
 {
 
-	fprintf(stderr, "FAIL\n");
+	printf("not ok %d - %s\n", curtest, assertion);
 
 	if (socktype == NULL)
-		fprintf(stderr, "%s(): assertion %s failed\n", func,
+		printf("# %s(): assertion %s failed\n", func,
 		    assertion);
 	else if (rest == NULL)
-		fprintf(stderr, "%s(%s): assertion %s failed\n", func,
+		printf("# %s(%s): assertion %s failed\n", func,
 		    socktype, assertion);
 	else
-		fprintf(stderr, "%s(%s, %s): assertion %s failed\n", func,
+		printf("# %s(%s, %s): assertion %s failed\n", func,
 		    socktype, rest, assertion);
 	exit(-1);
 }
@@ -107,6 +112,7 @@ test_evfilt_read(int kq, int fd[2], const char *socktype)
 	EV_SET(&ke, fd[0], EVFILT_READ, EV_ADD, 0, 0, NULL);
 	if (kevent(kq, &ke, 1, NULL, 0, NULL) == -1)
 		fail(errno, "kevent", socktype, "EVFILT_READ, EV_ADD");
+	OK("EVFILT_READ, EV_ADD");
 
 	/*
 	 * Confirm not readable to begin with, no I/O yet.
@@ -116,9 +122,11 @@ test_evfilt_read(int kq, int fd[2], const char *socktype)
 	i = kevent(kq, NULL, 0, &ke, 1, &ts);
 	if (i == -1)
 		fail(errno, "kevent", socktype, "EVFILT_READ");
+	OK("EVFILT_READ");
 	if (i != 0)
 		fail_assertion("kevent", socktype, "EVFILT_READ",
 		    "empty socket unreadable");
+	OK("empty socket unreadable");
 
 	/*
 	 * Write a byte to one end.
@@ -127,8 +135,10 @@ test_evfilt_read(int kq, int fd[2], const char *socktype)
 	len = write(fd[1], &ch, sizeof(ch));
 	if (len == -1)
 		fail(errno, "write", socktype, NULL);
+	OK("write one byte");
 	if (len != sizeof(ch))
 		fail_assertion("write", socktype, NULL, "write length");
+	OK("write one byte length");
 
 	/*
 	 * Other end should now be readable.
@@ -138,9 +148,11 @@ test_evfilt_read(int kq, int fd[2], const char *socktype)
 	i = kevent(kq, NULL, 0, &ke, 1, &ts);
 	if (i == -1)
 		fail(errno, "kevent", socktype, "EVFILT_READ");
+	OK("EVFILT_READ");
 	if (i != 1)
 		fail_assertion("kevent", socktype, "EVFILT_READ",
 		    "non-empty socket unreadable");
+	OK("non-empty socket unreadable");
 
 	/*
 	 * Read a byte to clear the readable state.
@@ -148,8 +160,10 @@ test_evfilt_read(int kq, int fd[2], const char *socktype)
 	len = read(fd[0], &ch, sizeof(ch));
 	if (len == -1)
 		fail(errno, "read", socktype, NULL);
+	OK("read one byte");
 	if (len != sizeof(ch))
 		fail_assertion("read", socktype, NULL, "read length");
+	OK("read one byte length");
 
 	/*
 	 * Now re-check for readability.
@@ -159,13 +173,16 @@ test_evfilt_read(int kq, int fd[2], const char *socktype)
 	i = kevent(kq, NULL, 0, &ke, 1, &ts);
 	if (i == -1)
 		fail(errno, "kevent", socktype, "EVFILT_READ");
+	OK("EVFILT_READ");
 	if (i != 0)
 		fail_assertion("kevent", socktype, "EVFILT_READ",
 		    "empty socket unreadable");
+	OK("empty socket unreadable");
 
 	EV_SET(&ke, fd[0], EVFILT_READ, EV_DELETE, 0, 0, NULL);
 	if (kevent(kq, &ke, 1, NULL, 0, NULL) == -1)
 		fail(errno, "kevent", socktype, "EVFILT_READ, EV_DELETE");
+	OK("EVFILT_READ, EV_DELETE");
 }
 
 static void
@@ -180,6 +197,7 @@ test_evfilt_write(int kq, int fd[2], const char *socktype)
 	EV_SET(&ke, fd[0], EVFILT_WRITE, EV_ADD, 0, 0, NULL);
 	if (kevent(kq, &ke, 1, NULL, 0, NULL) == -1)
 		fail(errno, "kevent", socktype, "EVFILT_WRITE, EV_ADD");
+	OK("EVFILE_WRITE, EV_ADD");
 
 	/*
 	 * Confirm writable to begin with, no I/O yet.
@@ -189,9 +207,11 @@ test_evfilt_write(int kq, int fd[2], const char *socktype)
 	i = kevent(kq, NULL, 0, &ke, 1, &ts);
 	if (i == -1)
 		fail(errno, "kevent", socktype, "EVFILT_WRITE");
+	OK("EVFILE_WRITE");
 	if (i != 1)
 		fail_assertion("kevent", socktype, "EVFILT_WRITE",
 		    "empty socket unwritable");
+	OK("empty socket unwritable");
 
 	/*
 	 * Write bytes into the socket until we can't write anymore.
@@ -200,8 +220,10 @@ test_evfilt_write(int kq, int fd[2], const char *socktype)
 	while ((len = write(fd[0], &ch, sizeof(ch))) == sizeof(ch)) {};
 	if (len == -1 && errno != EAGAIN && errno != ENOBUFS)
 		fail(errno, "write", socktype, NULL);
+	OK("write");
 	if (len != -1 && len != sizeof(ch))
 		fail_assertion("write", socktype, NULL, "write length");
+	OK("write length");
 
 	/*
 	 * Check to make sure the socket is no longer writable.
@@ -211,13 +233,16 @@ test_evfilt_write(int kq, int fd[2], const char *socktype)
 	i = kevent(kq, NULL, 0, &ke, 1, &ts);
 	if (i == -1)
 		fail(errno, "kevent", socktype, "EVFILT_WRITE");
+	OK("EVFILT_WRITE");
 	if (i != 0)
 		fail_assertion("kevent", socktype, "EVFILT_WRITE",
 		    "full socket writable");
+	OK("full socket writable");
 
 	EV_SET(&ke, fd[0], EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
 	if (kevent(kq, &ke, 1, NULL, 0, NULL) == -1)
 		fail(errno, "kevent", socktype, "EVFILT_WRITE, EV_DELETE");
+	OK("EVFILT_WRITE, EV_DELETE");
 }
 
 /*
@@ -229,9 +254,12 @@ main(int argc, char *argv[])
 {
 	int i, kq, sv[2];
 
+	printf("1..49\n");
+
 	kq = kqueue();
 	if (kq == -1)
 		fail(errno, "kqueue", NULL, NULL);
+	OK("kqueue()");
 
 	/*
 	 * Create a UNIX domain datagram socket, and attach/test/detach a
@@ -239,18 +267,23 @@ main(int argc, char *argv[])
 	 */
 	if (socketpair(PF_UNIX, SOCK_DGRAM, 0, sv) == -1)
 		fail(errno, "socketpair", "PF_UNIX, SOCK_DGRAM", NULL);
+	OK("socketpair() 1");
 
 	if (fcntl(sv[0], F_SETFL, O_NONBLOCK) != 0)
 		fail(errno, "fcntl", "PF_UNIX, SOCK_DGRAM", "O_NONBLOCK");
+	OK("fcntl() 1");
 	if (fcntl(sv[1], F_SETFL, O_NONBLOCK) != 0)
 		fail(errno, "fcntl", "PF_UNIX, SOCK_DGRAM", "O_NONBLOCK");
+	OK("fnctl() 2");
 
 	test_evfilt_read(kq, sv, "PF_UNIX, SOCK_DGRAM");
 
 	if (close(sv[0]) == -1)
 		fail(errno, "close", "PF_UNIX/SOCK_DGRAM", "sv[0]");
+	OK("close() 1");
 	if (close(sv[1]) == -1)
 		fail(errno, "close", "PF_UNIX/SOCK_DGRAM", "sv[1]");
+	OK("close() 2");
 
 #if 0
 	/*
@@ -285,18 +318,23 @@ main(int argc, char *argv[])
 	 */
 	if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1)
 		fail(errno, "socketpair", "PF_UNIX, SOCK_STREAM", NULL);
+	OK("socketpair() 2");
 
 	if (fcntl(sv[0], F_SETFL, O_NONBLOCK) != 0)
 		fail(errno, "fcntl", "PF_UNIX, SOCK_STREAM", "O_NONBLOCK");
+	OK("fcntl() 3");
 	if (fcntl(sv[1], F_SETFL, O_NONBLOCK) != 0)
 		fail(errno, "fcntl", "PF_UNIX, SOCK_STREAM", "O_NONBLOCK");
+	OK("fcntl() 4");
 
 	test_evfilt_read(kq, sv, "PF_UNIX, SOCK_STREAM");
 
 	if (close(sv[0]) == -1)
 		fail(errno, "close", "PF_UNIX/SOCK_STREAM", "sv[0]");
+	OK("close() 3");
 	if (close(sv[1]) == -1)
 		fail(errno, "close", "PF_UNIX/SOCK_STREAM", "sv[1]");
+	OK("close() 4");
 
 	/*
 	 * Create a UNIX domain stream socket, and attach/test/detach a
@@ -304,22 +342,27 @@ main(int argc, char *argv[])
 	 */
 	if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1)
 		fail(errno, "socketpair", "PF_UNIX, SOCK_STREAM", NULL);
+	OK("socketpair() 3");
 
 	if (fcntl(sv[0], F_SETFL, O_NONBLOCK) != 0)
 		fail(errno, "fcntl", "PF_UNIX, SOCK_STREAM", "O_NONBLOCK");
+	OK("fcntl() 5");
 	if (fcntl(sv[1], F_SETFL, O_NONBLOCK) != 0)
 		fail(errno, "fcntl", "PF_UNIX, SOCK_STREAM", "O_NONBLOCK");
+	OK("fcntl() 6");
 
 	test_evfilt_write(kq, sv, "PF_UNIX, SOCK_STREAM");
 
 	if (close(sv[0]) == -1)
 		fail(errno, "close", "PF_UNIX/SOCK_STREAM", "sv[0]");
+	OK("close() 5");
 	if (close(sv[1]) == -1)
 		fail(errno, "close", "PF_UNIX/SOCK_STREAM", "sv[1]");
+	OK("close() 6");
 
 	if (close(kq) == -1)
 		fail(errno, "close", "kq", NULL);
+	OK("close() 7");
 
-	printf("PASS\n");
 	return (0);
 }
diff --git a/tools/regression/sockets/kqueue/kqueue.t b/tools/regression/sockets/kqueue/kqueue.t
new file mode 100644
index 0000000..8bdfd03
--- /dev/null
+++ b/tools/regression/sockets/kqueue/kqueue.t
@@ -0,0 +1,10 @@
+#!/bin/sh
+# $FreeBSD$
+
+cd `dirname $0`
+
+executable=`basename $0 .t`
+
+make $executable 2>&1 > /dev/null
+
+exec ./$executable