MFC r326861-r326863:

Add some gmirror tests and fix indentation in existing tests.

9 files changed, 331 insertions, 13 deletions

diff --git a/tests/sys/geom/class/geom_subr.sh b/tests/sys/geom/class/geom_subr.sh
index b03ee43..3283d1b 100644
--- a/tests/sys/geom/class/geom_subr.sh
+++ b/tests/sys/geom/class/geom_subr.sh
@@ -20,6 +20,16 @@ attach_md()
 	echo $test_md
 }
 
+detach_md()
+{
+	local test_md unit
+
+	test_md=$1
+	unit=${test_md#md}
+	mdconfig -d -u $unit || exit
+	sed -i '' "/^${test_md}$/d" $TEST_MDS_FILE || exit
+}
+
 geom_test_cleanup()
 {
 	local test_md
@@ -38,6 +48,7 @@ if [ $(id -u) -ne 0 ]; then
 	echo '1..0 # SKIP tests must be run as root'
 	exit 0
 fi
+
 # If the geom class isn't already loaded, try loading it.
 if ! kldstat -q -m g_${class}; then
 	if ! geom ${class} load; then
diff --git a/tests/sys/geom/class/mirror/10_test.sh b/tests/sys/geom/class/mirror/10_test.sh
new file mode 100755
index 0000000..c818718
--- /dev/null
+++ b/tests/sys/geom/class/mirror/10_test.sh
@@ -0,0 +1,69 @@
+#!/bin/sh
+# $FreeBSD$
+
+# Test handling of read errors.
+
+. $(dirname $0)/conf.sh
+
+echo 1..3
+
+set -e
+
+ddbs=2048
+regreadfp="debug.fail_point.g_mirror_regular_request_read"
+m1=$(mktemp $base.XXXXXX)
+m2=$(mktemp $base.XXXXXX)
+
+dd if=/dev/random of=$m1 bs=$ddbs count=1024 >/dev/null 2>&1
+dd if=/dev/zero of=$m2 bs=$ddbs count=1024 >/dev/null 2>&1
+
+us0=$(attach_md -t vnode -f $m1)
+us1=$(attach_md -t vnode -f $m2)
+
+gmirror label $name /dev/$us0
+gmirror insert $name /dev/$us1
+devwait
+syncwait
+
+tmp1=$(mktemp $base.XXXXXX)
+tmp2=$(mktemp $base.XXXXXX)
+
+EIO=5
+# gmirror should retry a failed read from the other mirror.
+sysctl ${regreadfp}="1*return(${EIO})"
+dd if=/dev/mirror/$name of=$tmp1 iseek=256 bs=$ddbs count=1 >/dev/null 2>&1
+dd if=/dev/$us1 of=$tmp2 iseek=256 bs=$ddbs count=1 >/dev/null 2>&1
+sysctl ${regreadfp}='off'
+
+if cmp -s $tmp1 $tmp2; then
+	echo "ok 1"
+else
+	echo "not ok 1"
+fi
+
+# Make sure that one of the mirrors was marked broken.
+genid1=$(gmirror dump /dev/$us0 | awk '/^[[:space:]]*genid: /{print $2}')
+genid2=$(gmirror dump /dev/$us1 | awk '/^[[:space:]]*genid: /{print $2}')
+if [ $genid1 -eq $(($genid2 + 1)) -o $genid2 -eq $(($genid1 + 1)) ]; then
+	echo "ok 2"
+else
+	echo "not ok 2"
+fi
+
+# Force a retaste of the disconnected component.
+if [ $(gmirror status -s $name | awk '{print $3}') = $us0 ]; then
+	detach_md $us1
+	us1=$(attach_md -t vnode -f $m2)
+else
+	detach_md $us0
+	us0=$(attach_md -t vnode -f $m1)
+fi
+
+# Make sure that the component wasn't re-added to the gmirror.
+if [ $(gmirror status -s $name | wc -l) -eq 1 ]; then
+	echo "ok 3"
+else
+	echo "not ok 3"
+fi
+
+rm -f $m1 $m2 $tmp1 $tmp2
diff --git a/tests/sys/geom/class/mirror/11_test.sh b/tests/sys/geom/class/mirror/11_test.sh
new file mode 100755
index 0000000..284aa19
--- /dev/null
+++ b/tests/sys/geom/class/mirror/11_test.sh
@@ -0,0 +1,84 @@
+#!/bin/sh
+# $FreeBSD$
+
+# Test handling of read errors.
+
+. $(dirname $0)/conf.sh
+
+echo 1..4
+
+set -e
+
+ddbs=2048
+regreadfp="debug.fail_point.g_mirror_regular_request_read"
+m1=$(mktemp $base.XXXXXX)
+m2=$(mktemp $base.XXXXXX)
+
+dd if=/dev/random of=$m1 bs=$ddbs count=1024 >/dev/null 2>&1
+dd if=/dev/zero of=$m2 bs=$ddbs count=1024 >/dev/null 2>&1
+
+us0=$(attach_md -t vnode -f $m1)
+us1=$(attach_md -t vnode -f $m2)
+
+gmirror label $name /dev/$us0
+gmirror insert $name /dev/$us1
+devwait
+syncwait
+
+tmp1=$(mktemp $base.XXXXXX)
+tmp2=$(mktemp $base.XXXXXX)
+
+ENXIO=6
+# gmirror has special handling for ENXIO. It does not mark the failed component
+# as broken, allowing it to rejoin the mirror automatically when it appears.
+sysctl ${regreadfp}="1*return(${ENXIO})"
+dd if=/dev/mirror/$name of=$tmp1 iseek=512 bs=$ddbs count=1 >/dev/null 2>&1
+dd if=/dev/$us1 of=$tmp2 iseek=512 bs=$ddbs count=1 >/dev/null 2>&1
+sysctl ${regreadfp}='off'
+
+if cmp -s $tmp1 $tmp2; then
+	echo "ok 1"
+else
+	echo "not ok 1"
+fi
+
+# Verify that the genids still match after ENXIO.
+genid1=$(gmirror dump /dev/$us0 | awk '/^[[:space:]]*genid: /{print $2}')
+genid2=$(gmirror dump /dev/$us1 | awk '/^[[:space:]]*genid: /{print $2}')
+if [ $genid1 -eq $genid2 ]; then
+	echo "ok 2"
+else
+	echo "not ok 2"
+fi
+
+# Trigger a syncid bump.
+dd if=/dev/zero of=/dev/mirror/$name bs=$ddbs count=1 >/dev/null 2>&1
+
+# The ENXIO+write should have caused a syncid bump.
+syncid1=$(gmirror dump /dev/$us0 | awk '/^[[:space:]]*syncid: /{print $2}')
+syncid2=$(gmirror dump /dev/$us1 | awk '/^[[:space:]]*syncid: /{print $2}')
+if [ $syncid1 -eq $(($syncid2 + 1)) -o $syncid2 -eq $(($syncid1 + 1)) ]; then
+	echo "ok 3"
+else
+	echo "not ok 3"
+fi
+
+# Force a retaste of the disconnected component.
+if [ $(gmirror status -s $name | awk '{print $3}') = $us0 ]; then
+	detach_md $us1
+	us1=$(attach_md -t vnode -f $m2)
+else
+	detach_md $us0
+	us0=$(attach_md -t vnode -f $m1)
+fi
+
+# Make sure that the retaste caused the mirror to automatically be re-added.
+if [ $(gmirror status -s $name | wc -l) -eq 2 ]; then
+	echo "ok 4"
+else
+	echo "not ok 4"
+fi
+
+syncwait
+
+rm -f $m1 $m2 $tmp1 $tmp2
diff --git a/tests/sys/geom/class/mirror/12_test.sh b/tests/sys/geom/class/mirror/12_test.sh
new file mode 100755
index 0000000..07d69b6
--- /dev/null
+++ b/tests/sys/geom/class/mirror/12_test.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+# $FreeBSD$
+
+# Test handling of write errors.
+
+. $(dirname $0)/conf.sh
+
+echo 1..3
+
+set -e
+
+ddbs=2048
+regwritefp="debug.fail_point.g_mirror_regular_request_write"
+m1=$(mktemp $base.XXXXXX)
+m2=$(mktemp $base.XXXXXX)
+
+dd if=/dev/zero of=$m1 bs=$ddbs count=1024 >/dev/null 2>&1
+dd if=/dev/zero of=$m2 bs=$ddbs count=1024 >/dev/null 2>&1
+
+us0=$(attach_md -t vnode -f $m1)
+us1=$(attach_md -t vnode -f $m2)
+
+gmirror label $name /dev/$us0 /dev/$us1
+devwait
+
+tmp1=$(mktemp $base.XXXXXX)
+tmp2=$(mktemp $base.XXXXXX)
+dd if=/dev/random of=$tmp1 bs=$ddbs count=1 >/dev/null 2>&1
+
+EIO=5
+# gmirror should kick one of the mirrors out after hitting EIO.
+sysctl ${regwritefp}="1*return(${EIO})"
+dd if=$tmp1 of=/dev/mirror/$name bs=$ddbs count=1 >/dev/null 2>&1
+dd if=/dev/mirror/$name of=$tmp2 bs=$ddbs count=1 >/dev/null 2>&1
+sysctl ${regwritefp}='off'
+
+if cmp -s $tmp1 $tmp2; then
+	echo "ok 1"
+else
+	echo "not ok 1"
+fi
+
+# Make sure that one of the mirrors was marked broken.
+genid1=$(gmirror dump /dev/$us0 | awk '/^[[:space:]]*genid: /{print $2}')
+genid2=$(gmirror dump /dev/$us1 | awk '/^[[:space:]]*genid: /{print $2}')
+if [ $genid1 -eq $(($genid2 + 1)) -o $genid2 -eq $(($genid1 + 1)) ]; then
+	echo "ok 2"
+else
+	echo "not ok 2"
+fi
+
+# Force a retaste of the disconnected component.
+if [ $(gmirror status -s $name | awk '{print $3}') = $us0 ]; then
+	detach_md $us1
+	us1=$(attach_md -t vnode -f $m2)
+else
+	detach_md $us0
+	us0=$(attach_md -t vnode -f $m1)
+fi
+
+# Make sure that the component wasn't re-added to the gmirror.
+if [ $(gmirror status -s $name | wc -l) -eq 1 ]; then
+	echo "ok 3"
+else
+	echo "not ok 3"
+fi
+
+rm -f $m1 $m2 $tmp1 $tmp2
diff --git a/tests/sys/geom/class/mirror/13_test.sh b/tests/sys/geom/class/mirror/13_test.sh
new file mode 100755
index 0000000..4a66d65
--- /dev/null
+++ b/tests/sys/geom/class/mirror/13_test.sh
@@ -0,0 +1,81 @@
+#!/bin/sh
+# $FreeBSD$
+
+# Test handling of write errors.
+
+. $(dirname $0)/conf.sh
+
+echo 1..4
+
+set -e
+
+ddbs=2048
+regwritefp="debug.fail_point.g_mirror_regular_request_write"
+m1=$(mktemp $base.XXXXXX)
+m2=$(mktemp $base.XXXXXX)
+
+dd if=/dev/random of=$m1 bs=$ddbs count=1024 >/dev/null 2>&1
+dd if=/dev/zero of=$m2 bs=$ddbs count=1024 >/dev/null 2>&1
+
+us0=$(attach_md -t vnode -f $m1)
+us1=$(attach_md -t vnode -f $m2)
+
+gmirror label $name /dev/$us0 /dev/$us1
+devwait
+
+tmp1=$(mktemp $base.XXXXXX)
+tmp2=$(mktemp $base.XXXXXX)
+
+dd if=/dev/random of=$tmp1 bs=$ddbs count=1 >/dev/null 2>&1
+
+ENXIO=6
+# gmirror has special handling for ENXIO. It does not mark the failed component
+# as broken, allowing it to rejoin the mirror automatically when it appears.
+sysctl ${regwritefp}="1*return(${ENXIO})"
+dd if=$tmp1 of=/dev/mirror/$name bs=$ddbs count=1 >/dev/null 2>&1
+dd if=/dev/mirror/$name of=$tmp2 bs=$ddbs count=1 >/dev/null 2>&1
+sysctl ${regwritefp}='off'
+
+if cmp -s $tmp1 $tmp2; then
+	echo "ok 1"
+else
+	echo "not ok 1"
+fi
+
+# Verify that the genids still match after ENXIO.
+genid1=$(gmirror dump /dev/$us0 | awk '/^[[:space:]]*genid: /{print $2}')
+genid2=$(gmirror dump /dev/$us1 | awk '/^[[:space:]]*genid: /{print $2}')
+if [ $genid1 -eq $genid2 ]; then
+	echo "ok 2"
+else
+	echo "not ok 2"
+fi
+
+# The ENXIO should have caused a syncid bump.
+syncid1=$(gmirror dump /dev/$us0 | awk '/^[[:space:]]*syncid: /{print $2}')
+syncid2=$(gmirror dump /dev/$us1 | awk '/^[[:space:]]*syncid: /{print $2}')
+if [ $syncid1 -eq $(($syncid2 + 1)) -o $syncid2 -eq $(($syncid1 + 1)) ]; then
+	echo "ok 3"
+else
+	echo "not ok 3"
+fi
+
+# Force a retaste of the disconnected component.
+if [ $(gmirror status -s $name | awk '{print $3}') = $us0 ]; then
+	detach_md $us1
+	us1=$(attach_md -t vnode -f $m2)
+else
+	detach_md $us0
+	us0=$(attach_md -t vnode -f $m1)
+fi
+
+# Make sure that the retaste caused the mirror to automatically be re-added.
+if [ $(gmirror status -s $name | wc -l) -eq 2 ]; then
+	echo "ok 4"
+else
+	echo "not ok 4"
+fi
+
+syncwait
+
+rm -f $m1 $m2 $tmp1 $tmp2
diff --git a/tests/sys/geom/class/mirror/8_test.sh b/tests/sys/geom/class/mirror/8_test.sh
index 9eefc9d..d288190 100644
--- a/tests/sys/geom/class/mirror/8_test.sh
+++ b/tests/sys/geom/class/mirror/8_test.sh
@@ -35,9 +35,7 @@ devwait # This will take kern.geom.mirror.timeout seconds.
 
 # Re-attach the second mirror and wait for it to synchronize.
 us1=$(attach_md -t vnode -f $m2) || exit 1
-while [ $(gmirror status $name | grep ACTIVE | wc -l) -ne 2 ]; do
-    sleep 1
-done
+syncwait
 
 # Verify the two mirrors are identical. Destroy the gmirror first so that
 # the mirror metadata is wiped; otherwise the metadata blocks will fail
@@ -45,9 +43,9 @@ done
 # command instead.
 gmirror destroy $name
 if cmp -s ${m1} ${m2}; then
-    echo "ok 1"
+	echo "ok 1"
 else
-    echo "not ok 1"
+	echo "not ok 1"
 fi
 
 rm -f $m1 $m2
diff --git a/tests/sys/geom/class/mirror/9_test.sh b/tests/sys/geom/class/mirror/9_test.sh
index 072499f..c7af15b 100644
--- a/tests/sys/geom/class/mirror/9_test.sh
+++ b/tests/sys/geom/class/mirror/9_test.sh
@@ -33,9 +33,7 @@ sysctl debug.fail_point.g_mirror_metadata_write='off' || exit 1
 # Replace the broken mirror, and then stop the gmirror.
 gmirror forget $name || exit 1
 gmirror insert $name /dev/$us2 || exit 1
-while [ $(gmirror status $name | grep ACTIVE | wc -l) -ne 2 ]; do
-    sleep 1
-done
+syncwait
 gmirror stop $name || exit 1
 
 # Restart the gmirror on the original two mirrors. One of them is broken,
@@ -49,14 +47,12 @@ dd if=/dev/random of=/dev/mirror/$name bs=$ddbs count=1 >/dev/null 2>&1
 # the metadata blocks will fail the comparison. It would be nice to do this
 # with a "gmirror verify" command instead.
 gmirror activate $name /dev/$us2 || exit 1
-while [ $(gmirror status $name | grep ACTIVE | wc -l) -ne 2 ]; do
-    sleep 1
-done
+syncwait
 gmirror destroy $name || exit 1
 if cmp -s $m1 $m3; then
-    echo "ok 1"
+	echo "ok 1"
 else
-    echo "not ok 1"
+	echo "not ok 1"
 fi
 
 rm -f $m1 $m2 $m3
diff --git a/tests/sys/geom/class/mirror/Makefile b/tests/sys/geom/class/mirror/Makefile
index 66f9249..095fb90 100644
--- a/tests/sys/geom/class/mirror/Makefile
+++ b/tests/sys/geom/class/mirror/Makefile
@@ -13,6 +13,10 @@ TAP_TESTS_SH+=	6_test
 TAP_TESTS_SH+=	7_test
 TAP_TESTS_SH+=	8_test
 TAP_TESTS_SH+=	9_test
+TAP_TESTS_SH+=	10_test
+TAP_TESTS_SH+=	11_test
+TAP_TESTS_SH+=	12_test
+TAP_TESTS_SH+=	13_test
 
 ${PACKAGE}FILES+=		conf.sh
 
diff --git a/tests/sys/geom/class/mirror/conf.sh b/tests/sys/geom/class/mirror/conf.sh
index d8595f1..1cb785e 100644
--- a/tests/sys/geom/class/mirror/conf.sh
+++ b/tests/sys/geom/class/mirror/conf.sh
@@ -12,4 +12,11 @@ gmirror_test_cleanup()
 }
 trap gmirror_test_cleanup ABRT EXIT INT TERM
 
+syncwait()
+{
+	while $(gmirror status -s $name | grep -q SYNCHRONIZING); do
+		sleep 0.1;
+	done
+}
+
 . `dirname $0`/../geom_subr.sh