diff options
Diffstat (limited to 'lib/libc_r/test/verify')
| -rwxr-xr-x | lib/libc_r/test/verify | 505 |
1 files changed, 505 insertions, 0 deletions
diff --git a/lib/libc_r/test/verify b/lib/libc_r/test/verify new file mode 100755 index 0000000..bf8d94a --- /dev/null +++ b/lib/libc_r/test/verify @@ -0,0 +1,505 @@ +#!/usr/bin/perl -w +#-*-mode:perl-*- +############################################################################# +# +# Copyright (C) 1999-2000 Jason Evans <jasone@freebsd.org>. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice(s), this list of conditions and the following disclaimer as +# the first lines of this file unmodified other than the possible +# addition of one or more copyright notices. +# 2. Redistributions in binary form must reproduce the above copyright +# notice(s), this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +############################################################################# +# +# Test harness. +# +# $FreeBSD$ +# +############################################################################# + +# Shut off buffering. +select(STDOUT); +$| = 1; + +# +# Parse command-line arguments. +# +use Getopt::Long; +Getopt::Long::config("bundling"); # Allow -hv rather than forcing -h -v. + +# Set option defaults for optional arguments. +$opt_help = 0; +$opt_verbose = 0; +$opt_quiet = 0; +$opt_srcdir = "."; +$opt_ustats = 0; +$opt_zero = 0; + +$opt_retval = +&GetOptions("h|help" => \$opt_help, + "v|verbose" => \$opt_verbose, + "q|quiet" => \$opt_quiet, + "s|srcdir=s" => \$opt_srcdir, + "u|ustats" => \$opt_ustats, + "z|zero" => \$opt_zero + ); + +$mode = "d"; +for ($i = 0; $i <= $#ARGV; $i++) +{ + if (($ARGV[$i] eq "-s") || ($ARGV[$i] eq "--sequence")) + { + $mode = "s"; + } + elsif (($ARGV[$i] eq "-d") || ($ARGV[$i] eq "--diff")) + { + $mode = "d"; + } + elsif (-x $ARGV[$i]) + { + if ($mode eq "s") + { + @STESTS = (@STESTS, $ARGV[$i]); + } + else + { + @DTESTS = (@DTESTS, $ARGV[$i]); + } + } + else + { + if (!$opt_quiet) + { + print STDERR "Cannot run \"$ARGV[$i]\"\n"; + } + } +} + +if ($opt_help) +{ + &usage(); + exit(0); +} + +if ($opt_retval == 0) +{ + &usage(); + exit 1; +} + +if ($opt_verbose && $opt_quiet) +{ + print STDERR "-v and -q are incompatible\n"; + &usage(); + exit 1; +} + +if ($#STESTS + $#DTESTS + 2 == 0) +{ + print STDERR "No tests specified\n"; + &usage(); + exit 1; +} + +if ($opt_verbose) +{ + print STDERR "Option values: h:$opt_help, v:$opt_verbose, q:$opt_quiet, " + . "u:$opt_ustats\n"; + printf STDERR "Sequence tests (%d total): @STESTS\n", $#STESTS + 1; + printf STDERR "Diff tests (%d total): @DTESTS\n", $#DTESTS + 1; +} + +# +# Create and print header. +# +@TSTATS = +( + "--------------------------------------------------------------------------\n", + "Test c_user c_system c_total chng\n", + " passed/FAILED h_user h_system h_total %% chng\n" + ); + +if (!$opt_quiet) +{ + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } +} + +# +# Run sequence test(s). +# +$total_utime = 0.0; # Total user time. +$total_stime = 0.0; # Total system time. +$total_hutime = 0.0; # Total historical user time. +$total_hstime = 0.0; # Total historical system time. +$total_ntime = 0.0; # Total time for tests that have historical data. + +foreach $test (@STESTS) +{ + # sequence mode. + $okay = 1; + + ($okay, $utime, $stime) = &run_test($test); + + if (open (STEST_OUT, "<./$test.out")) + { + $num_failed_subtests = 0; + + $_ = <STEST_OUT>; + + if ($_ =~ /1\.\.(\d+)/) + { + $num_subtests = $1; + } + else + { + $num_subtests = 0; + $okay = 0; + if ($opt_verbose) + { + print STDERR "Malformed 1..n line: \"$_\"\n"; + } + } + for ($subtest = 1; $subtest <= $num_subtests; $subtest++) + { + while (defined($line = <STEST_OUT>)) + { + if ($line =~ /^not\s+ok\s+(\d+)?/) + { + $not = 1; + $test_num = $1; + last; + } + elsif ($line =~ /^ok\s+(\d+)?/) + { + $not = 0; + $test_num = $1; + last; + } + } + if (defined($line)) + { + if (defined($test_num) && ($test_num != $subtest)) + { + # There was no output printed for one or more tests. + for (; $subtest < $test_num; $subtest++) + { + $num_failed_subtests++; + } + } + if ($not) + { + $num_failed_subtests++; + } + } + else + { + for (; $subtest <= $num_subtests; $subtest++) + { + $num_failed_subtests++; + } + } + } + + if (0 < $num_failed_subtests) + { + $okay = 0; + } + } + else + { + if (!$opt_quiet) + { + print STDERR "Cannot open output file \"$test.out\"\n"; + } + exit 1; + } + + ($hutime, $hstime) = &print_stats($test, $okay, + $num_failed_subtests, $num_subtests, + $utime, $stime); + $total_hutime += $hutime; + $total_hstime += $hstime; + + if ($okay) + { + $total_utime += $utime; + $total_stime += $stime; + } + else + { + @FAILED_TESTS = (@FAILED_TESTS, $test); + } + + # If there were historical data, add the run time to the total time to + # compare against the historical run time. + if (0 < ($hutime + $hstime)) + { + $total_ntime += $utime + $stime; + } +} +foreach $test (@DTESTS) +{ + # Diff mode. + $okay = 1; + + ($okay, $utime, $stime) = &run_test($test); + + if (-e "./$test.out" && -e "$opt_srcdir/$test.exp") + { + `diff ./$test.out $opt_srcdir/$test.exp > ./$test.diff 2>&1`; + if ($?) + { + # diff returns non-zero if there is a difference. + $okay = 0; + } + } + else + { + $okay = 0; + if ($opt_verbose) + { + print STDERR + "Nonexistent expected output file \"$opt_srcdir/$test.exp\"\n"; + print STDERR "\$opt_srcdir is \"$opt_srcdir\"\n"; + } + } + + ($hutime, $hstime) = &print_stats($test, $okay, 0, 0, $utime, $stime); + $total_hutime += $hutime; + $total_hstime += $hstime; + + if ($okay) + { + $total_utime += $utime; + $total_stime += $stime; + } + else + { + @FAILED_TESTS = (@FAILED_TESTS, $test); + } + + # If there were historical data, add the run time to the total time to + # compare against the historical run time. + if (0 < ($hutime + $hstime)) + { + $total_ntime += $utime + $stime; + } +} + +# Print summary stats. +$tt_str = sprintf ("%d / %d passed (%5.2f%%%%)", + ($#STESTS + $#DTESTS + 2) - ($#FAILED_TESTS + 1), + $#STESTS + $#DTESTS + 2, + (($#STESTS + $#DTESTS + 2) - ($#FAILED_TESTS + 1)) + / ($#STESTS + $#DTESTS + 2) * 100); + +$t_str = sprintf ("Totals %7.2f %7.2f %7.2f" + . " %7.2f\n" + . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n", + $total_utime, $total_stime, $total_utime + $total_stime, + ($total_ntime - ($total_hutime + $total_hstime)), + $tt_str . ' ' x (40 - length($tt_str)), + $total_hutime, $total_hstime, $total_hutime + $total_hstime, + ($total_hutime + $total_hstime == 0.0) ? 0.0 : + (($total_ntime + - ($total_hutime + $total_hstime)) + / ($total_hutime + $total_hstime) * 100)); + +@TSTATS = ("--------------------------------------------------------------------------\n", + $t_str, + "--------------------------------------------------------------------------\n" + ); +if (!$opt_quiet) +{ + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } +} + +# End of main execution. + +sub run_test +{ + my ($test) = @_; + my ($okay) = 1; + my ($tutime, $tstime); + my ($utime, $stime, $cutime, $cstime); + my (@TSTATS); + my ($t_str); + + @TSTATS = ("--------------------------------------------------------------------------\n"); + + $t_str = sprintf ("%s%s", $test, ' ' x (40 - length($test))); + @TSTATS = (@TSTATS, $t_str); + @STATS = (@STATS, @TSTATS); + if (!$opt_quiet) + { + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } + } + + ($utime, $stime, $cutime, $cstime) = times; + `./$test $opt_srcdir > ./$test.out 2>&1`; + ($utime, $stime, $tutime, $tstime) = times; + + # Subtract the before time from the after time. + $tutime -= $cutime; + $tstime -= $cstime; + + if ($opt_zero) + { + if ($?) + { + $okay = 0; + if ($opt_verbose) + { + print STDERR "\"./$test > ./$test.out 2>&1\" returned $?\n"; + } + } + } + + return ($okay, $tutime, $tstime); +} + +sub print_stats +{ + my ($test, $okay, $failed_subtests, $subtests, $utime, $stime) = @_; + my ($hutime, $hstime); +# my (TEST_PERF); + my (@TSTATS); + my ($t_str, $pass_str); + + $pass_str = $okay ? "passed" : "*** FAILED ***"; + if ((0 != $subtests) && (!$okay)) + { + $pass_str = $pass_str . " ($failed_subtests/$subtests failed)"; + } + $pass_str = $pass_str . ' ' x (39 - length($pass_str)); + + if (-r "$test.perf") + { + if (!open (TEST_PERF, "<./$test.perf")) + { + print STDERR "Unable to open \"./$test.perf\"\n"; + exit 1; + } + $_ = <TEST_PERF>; + + ($hutime, $hstime) = split; + close TEST_PERF; + + $t_str = sprintf (" %7.2f %7.2f %7.2f %7.2f\n" + . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n", + $utime, $stime, $utime + $stime, + ($utime + $stime) - ($hutime + $hstime), + $pass_str, + $hutime, $hstime, $hutime + $hstime, + (($hutime + $hstime) == 0.0) ? 0.0 : + ((($utime + $stime) - ($hutime + $hstime)) + / ($hutime + $hstime) * 100)); + } + else + { + $hutime = 0.0; + $hstime = 0.0; + + $t_str = sprintf (" %7.2f %7.2f %7.2f \n" + . " %s\n", + $utime, $stime, $utime + $stime, + $pass_str); + } + @TSTATS = ($t_str); + if (!$opt_quiet) + { + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } + } + + if ($okay && $opt_ustats) + { + if (!open (TEST_PERF, ">./$test.perf")) + { + if (!$opt_quiet) + { + print STDERR "Unable to update \"$test.perf\"\n"; + } + } + else + { + print TEST_PERF "$utime $stime\n"; + close TEST_PERF; + } + } + + return ($hutime, $hstime); +} + +sub usage +{ + print <<EOF; +$0 usage: + $0 [<options>] -- {[-s | -d | --sequence | --diff] <test>+}+ + + Option | Description + --------------+------------------------------------------------------------- + -h --help | Print usage and exit. + -v --verbose | Verbose (incompatible with quiet). + -q --quiet | Quiet (incompatible with verbose). + -s --srcdir | Path to source tree (default is "."). + -u --ustats | Update historical statistics (stored in "<test>.perf". + -z --zero | Consider non-zero exit code to be an error. + --------------+------------------------------------------------------------- + + Flag | Description + --------------+------------------------------------------------------------- + -s --sequence | Sequence mode (default). + | Output to stdout of the following form is expected: + | + | ----------------- + | 1..<n> + | {not} ok [1] + | {not} ok [2] + | ... + | {not} ok [n] + | ----------------- + | + | 1 <= <n> < 2^31 + | + | Lines which do not match the patterns shown above are + | ignored, except that the 1..<n> construct must be the first + | line of output seen. + | + -d --diff | Diff mode (incompatible with sequence mode). + | If <test>.exp exists, diff it with the output from <test> to + | determine success or failure. + --------------+------------------------------------------------------------- +EOF +} |
