Improve the RC framework for the clean booting/shutdown of Jails:

1. Feature: for flexibility reasons and as a prerequisite to clean
   shutdowns, allow the configuration of a stop/shutdown command
   via rc.conf variable "jail_<name>_exec_stop" in addition to the
   start/boot command (rc.conf variable "jail_<name>_exec_start"). For
   backward compatibility reasons, rc.conf variable "jail_<name>_exec"
   is still supported, too.

2. Debug: Add the used boot/shutdown commands to the debug output of
   the /etc/rc.d/jail script, too.

3. Security: Run the Jail start/boot command in a cleaned environment
   to not leak information from the host to the Jail during startup.

4. Feature: Run the Jail stop/shutdown command "jail_<name>_exec_stop" on
   "/etc/rc.d/jail stop <name>" to allow a graceful shutdown of the Jail
   before its processes are just killed.

5. Bugfix: When killing the remaining Jail processes give the processes
   time to actually perform their termination sequence. Without this the
   subsequent umount(8) operations usually fail because the resources
   are still in use. Additionally, if after trying to TERM-inate the
   processes there are still processes hanging around, finally just KILL
   them.

6. Bugfix: In rc.shutdown, if running inside a Jail, skip the /etc/rc.d/*
   scripts which are flagged with the KEYWORD "nojail" to allow the
   correct operation of rc.shutdown under jail_<name>_exec_stop="/bin/sh
   /etc/rc.shutdown". This is analogous to what /etc/rc does inside a Jail.

Now the following typical host-configuration for two Jails works as
expected and correctly boots and shutdowns the Jails:

-----------------------------------------------------------
#  /etc/rc.conf:
jail_enable="YES"
jail_list="foo bar"
jail_foo_rootdir="/j/foo"
jail_foo_hostname="foo.example.com"
jail_foo_ip="192.168.0.1"
jail_foo_devfs_enable="YES"
jail_foo_mount_enable="YES"
jail_foo_exec_start="/bin/sh /etc/rc"
jail_foo_exec_stop="/bin/sh /etc/rc.shutdown"
jail_bar_rootdir="/j/bar"
jail_bar_hostname="bar.example.com"
jail_bar_ip="192.168.0.2"
jail_bar_devfs_enable="YES"
jail_bar_mount_enable="YES"
jail_bar_exec_start="/path/to/kjailer -v"
jail_bar_exec_stop="/bin/sh -c 'killall kjailer && sleep 60'"
-----------------------------------------------------------
#  /etc/fstab.foo
/v/foo /j/foo/v/foo nullfs rw 0 0
-----------------------------------------------------------
#  /etc/fstab.bar
/v/bar /j/bar/v/bar nullfs rw 0 0
-----------------------------------------------------------

Reviewed by:	freebsd-hackers
MFC after:	2 weeks

1 files changed, 25 insertions, 3 deletions

diff --git a/etc/rc.d/jail b/etc/rc.d/jail
index 38d20c6..535a719 100644
--- a/etc/rc.d/jail
+++ b/etc/rc.d/jail
@@ -34,7 +34,21 @@ init_variables()
 	eval jail_hostname=\"\$jail_${_j}_hostname\"
 	eval jail_ip=\"\$jail_${_j}_ip\"
 	eval jail_exec=\"\$jail_${_j}_exec\"
-	[ -z "${jail_exec}" ] && jail_exec="/bin/sh /etc/rc"
+	eval jail_exec_start=\"\$jail_${_j}_exec_start\"
+	eval jail_exec_stop=\"\$jail_${_j}_exec_stop\"
+	if [ -n "${jail_exec}" ]; then
+		#   simple/backward-compatible execution
+		jail_exec_start="${jail_exec}"
+		jail_exec_stop=""
+	else
+		#   flexible execution
+		if [ -z "${jail_exec_start}" ]; then
+			jail_exec_start="/bin/sh /etc/rc"
+			if [ -z "${jail_exec_stop}" ]; then
+				jail_exec_stop="/bin/sh /etc/rc.shutdown"
+			fi
+		fi
+	fi
 
 	# The default jail ruleset will be used by rc.subr if none is specified.
 	eval jail_ruleset=\"\$jail_${_j}_devfs_ruleset\"
@@ -65,6 +79,8 @@ init_variables()
 	debug "$_j procdir: $jail_procdir"
 	debug "$_j ruleset: $jail_ruleset"
 	debug "$_j fstab: $jail_fstab"
+	debug "$_j exec start: $jail_exec_start"
+	debug "$_j exec stop: $jail_exec_stop"
 }
 
 # set_sysctl rc_knob mib msg
@@ -177,8 +193,8 @@ jail_start()
 			fi
 		fi
 		_tmp_jail=${_tmp_dir}/jail.$$
-		jail -i ${jail_rootdir} ${jail_hostname} \
-			${jail_ip} ${jail_exec} > ${_tmp_jail} 2>&1
+		eval jail -l -U root -i ${jail_rootdir} ${jail_hostname} \
+			${jail_ip} ${jail_exec_start} > ${_tmp_jail} 2>&1
 		[ "$?" -eq 0 ] && echo -n " $jail_hostname"
 		_jail_id=$(head -1 ${_tmp_jail})
 		tail +2 ${_tmp_jail} >${jail_rootdir}/var/log/console.log
@@ -198,7 +214,13 @@ jail_stop()
 			_jail_id=$(cat /var/run/jail_${_jail}.id)
 			if [ ! -z "${_jail_id}" ]; then
 				init_variables $_jail
+				if [ -n "${jail_exec_stop}" ]; then
+					eval env -i /usr/sbin/jexec ${_jail_id} ${jail_exec_stop} \
+						>> ${jail_rootdir}/var/log/console.log 2>&1
+				fi
 				killall -j ${_jail_id} -TERM > /dev/null 2>&1
+				sleep 1
+				killall -j ${_jail_id} -KILL > /dev/null 2>&1
 				jail_umount_fs
 				echo -n " $jail_hostname"
 			fi