Fix unintended KBI change from r264905. Add _fib versions of

ifa_ifwithnet() and ifa_ifwithdstaddr()  The legacy functions will call the
_fib() versions with RT_ALL_FIBS, preserving legacy behavior.

sys/net/if_var.h
sys/net/if.c
	Add legacy-compatible functions as described above.  Ensure legacy
	behavior when RT_ALL_FIBS is passed as fibnum.

sys/netinet/in_pcb.c
sys/netinet/ip_output.c
sys/netinet/ip_options.c
sys/net/route.c
sys/net/rtsock.c
sys/netinet6/nd6.c
	Call with _fib() functions if we must use a specific fib, or the
	legacy functions otherwise.

tests/sys/netinet/fibs_test.sh
tests/sys/netinet/udp_dontroute.c
	Improve the udp_dontroute test.  The bug that this test exercises is
	that ifa_ifwithnet() will return the wrong address, if multiple
	interfaces have addresses on the same subnet but with different
	fibs.  The previous version of the test only considered one possible
	failure mode: that ifa_ifwithnet_fib() might fail to find any
	suitable address at all.  The new version also checks whether
	ifa_ifwithnet_fib() finds the correct address by checking where the
	ARP request goes.

Reported by:	bz, hrs
Reviewed by:	hrs
MFC after:	1 week
X-MFC-with:	264905
Sponsored by:	Spectra Logic

2 files changed, 54 insertions, 15 deletions

diff --git a/tests/sys/netinet/fibs_test.sh b/tests/sys/netinet/fibs_test.sh
index d8b18e2..72ebcfc 100755
--- a/tests/sys/netinet/fibs_test.sh
+++ b/tests/sys/netinet/fibs_test.sh
@@ -369,25 +369,38 @@ udp_dontroute_body()
 	atf_expect_fail "kern/187553 Source address selection for UDP packets with SO_DONTROUTE uses the default FIB"
 	# Configure the TAP interface to use an RFC5737 nonrouteable address
 	# and a non-default fib
-	ADDR="192.0.2.2"
+	ADDR0="192.0.2.2"
+	ADDR1="192.0.2.3"
 	SUBNET="192.0.2.0"
 	MASK="24"
 	# Use a different IP on the same subnet as the target
 	TARGET="192.0.2.100"
+	SRCDIR=`atf_get_srcdir`
 
 	# Check system configuration
 	if [ 0 != `sysctl -n net.add_addr_allfibs` ]; then
 		atf_skip "This test requires net.add_addr_allfibs=0"
 	fi
-	get_fibs 1
+	get_fibs 2
 
-	# Configure a TAP interface
-	setup_tap ${FIB0} ${ADDR} ${MASK}
+	# Configure the TAP interfaces
+	setup_tap ${FIB0} ${ADDR0} ${MASK}
+	TARGET_TAP=${TAP}
+	setup_tap ${FIB1} ${ADDR1} ${MASK}
 
 	# Send a UDP packet with SO_DONTROUTE.  In the failure case, it will
-	# return ENETUNREACH
-	SRCDIR=`atf_get_srcdir`
-	atf_check -o ignore setfib ${FIB0} ${SRCDIR}/udp_dontroute ${TARGET}
+	# return ENETUNREACH, or send the packet to the wrong tap
+	atf_check -o ignore setfib ${FIB0} \
+		${SRCDIR}/udp_dontroute ${TARGET} /dev/${TARGET_TAP}
+	cleanup_tap
+
+	# Repeat, but this time target the other tap
+	setup_tap ${FIB0} ${ADDR0} ${MASK}
+	setup_tap ${FIB1} ${ADDR1} ${MASK}
+	TARGET_TAP=${TAP}
+
+	atf_check -o ignore setfib ${FIB1} \
+		${SRCDIR}/udp_dontroute ${TARGET} /dev/${TARGET_TAP}
 }
 
 udp_dontroute_cleanup()
@@ -467,4 +480,5 @@ cleanup_tap()
 	for TAPD in `cat "tap_devices_to_cleanup"`; do
 		ifconfig ${TAPD} destroy
 	done
+	rm "tap_devices_to_cleanup"
 }
diff --git a/tests/sys/netinet/udp_dontroute.c b/tests/sys/netinet/udp_dontroute.c
index 1e162a1..79421fd 100644
--- a/tests/sys/netinet/udp_dontroute.c
+++ b/tests/sys/netinet/udp_dontroute.c
@@ -39,9 +39,11 @@
 
 #include <err.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 
 /*
  * Sends a single UDP packet to the provided address, with SO_DONTROUTE set
@@ -51,23 +53,31 @@ int
 main(int argc, char **argv)
 {
 	struct sockaddr_in dst;
-	int s;
+	int s, t;
 	int opt;
 	int ret;
-	const char* buf = "Hello, World!";
+	ssize_t len;
+	const char* sendbuf = "Hello, World!";
+	const size_t buflen = 80;
+	char recvbuf[buflen];
 
-	if (argc != 2) {
-		fprintf(stderr, "Usage: %s ip_address\n", argv[0]);
+	if (argc != 3) {
+		fprintf(stderr, "Usage: %s ip_address tapdev\n", argv[0]);
 		exit(2);
 	}
+
+	t = open(argv[2], O_RDWR | O_NONBLOCK);
+	if (t < 0)
+		err(EXIT_FAILURE, "open");
+
 	s = socket(PF_INET, SOCK_DGRAM, 0);
 	if (s < 0)
-		err(errno, "socket");
+		err(EXIT_FAILURE, "socket");
 	opt = 1;
 
 	ret = setsockopt(s, SOL_SOCKET, SO_DONTROUTE, &opt, sizeof(opt));
 	if (ret == -1)
-		err(errno, "setsockopt(SO_DONTROUTE)");
+		err(EXIT_FAILURE, "setsockopt(SO_DONTROUTE)");
 
 	dst.sin_len = sizeof(dst);
 	dst.sin_family = AF_INET;
@@ -77,10 +87,25 @@ main(int argc, char **argv)
 		fprintf(stderr, "Invalid address: %s\n", argv[1]);
 		exit(2);
 	}
-	ret = sendto(s, buf, strlen(buf), 0, (struct sockaddr*)&dst,
+	ret = sendto(s, sendbuf, strlen(sendbuf), 0, (struct sockaddr*)&dst,
 	    dst.sin_len);
 	if (ret == -1)
-		err(errno, "sendto");
+		err(EXIT_FAILURE, "sendto");
+
+	/* Verify that the packet went to the desired tap device */
+
+	len = read(t, recvbuf, buflen);
+	if (len == 0)
+		errx(EXIT_FAILURE, "read returned EOF");
+	else if (len < 0 && errno == EAGAIN)
+		errx(EXIT_FAILURE, "Did not receive any packets");
+	else if (len < 0)
+		err(EXIT_FAILURE, "read");
 
+	/*
+	 * If read returned anything at all, consider it a success.  The packet
+	 * should be an Ethernet frame containing an ARP request for
+	 * ip_address.  We won't bother to decode it
+	 */
 	return (0);
 }