Improve the RC framework for the clean booting/shutdown of Jails:

1. Feature: for flexibility reasons and as a prerequisite to clean
   shutdowns, allow the configuration of a stop/shutdown command
   via rc.conf variable "jail_<name>_exec_stop" in addition to the
   start/boot command (rc.conf variable "jail_<name>_exec_start"). For
   backward compatibility reasons, rc.conf variable "jail_<name>_exec"
   is still supported, too.

2. Debug: Add the used boot/shutdown commands to the debug output of
   the /etc/rc.d/jail script, too.

3. Security: Run the Jail start/boot command in a cleaned environment
   to not leak information from the host to the Jail during startup.

4. Feature: Run the Jail stop/shutdown command "jail_<name>_exec_stop" on
   "/etc/rc.d/jail stop <name>" to allow a graceful shutdown of the Jail
   before its processes are just killed.

5. Bugfix: When killing the remaining Jail processes give the processes
   time to actually perform their termination sequence. Without this the
   subsequent umount(8) operations usually fail because the resources
   are still in use. Additionally, if after trying to TERM-inate the
   processes there are still processes hanging around, finally just KILL
   them.

6. Bugfix: In rc.shutdown, if running inside a Jail, skip the /etc/rc.d/*
   scripts which are flagged with the KEYWORD "nojail" to allow the
   correct operation of rc.shutdown under jail_<name>_exec_stop="/bin/sh
   /etc/rc.shutdown". This is analogous to what /etc/rc does inside a Jail.

Now the following typical host-configuration for two Jails works as
expected and correctly boots and shutdowns the Jails:

-----------------------------------------------------------
#  /etc/rc.conf:
jail_enable="YES"
jail_list="foo bar"
jail_foo_rootdir="/j/foo"
jail_foo_hostname="foo.example.com"
jail_foo_ip="192.168.0.1"
jail_foo_devfs_enable="YES"
jail_foo_mount_enable="YES"
jail_foo_exec_start="/bin/sh /etc/rc"
jail_foo_exec_stop="/bin/sh /etc/rc.shutdown"
jail_bar_rootdir="/j/bar"
jail_bar_hostname="bar.example.com"
jail_bar_ip="192.168.0.2"
jail_bar_devfs_enable="YES"
jail_bar_mount_enable="YES"
jail_bar_exec_start="/path/to/kjailer -v"
jail_bar_exec_stop="/bin/sh -c 'killall kjailer && sleep 60'"
-----------------------------------------------------------
#  /etc/fstab.foo
/v/foo /j/foo/v/foo nullfs rw 0 0
-----------------------------------------------------------
#  /etc/fstab.bar
/v/bar /j/bar/v/bar nullfs rw 0 0
-----------------------------------------------------------

Reviewed by:	freebsd-hackers
MFC after:	2 weeks

3 files changed, 30 insertions, 5 deletions

diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf
index 835e10f..b8b08e0 100644
--- a/etc/defaults/rc.conf
+++ b/etc/defaults/rc.conf
@@ -493,7 +493,8 @@ jail_sysvipc_allow="NO"	# Allow SystemV IPC use from within a jail
 #jail_example_rootdir="/usr/jail/default"	# Jail's root directory
 #jail_example_hostname="default.domain.com"	# Jail's hostname
 #jail_example_ip="192.168.0.10"			# Jail's IP number
-#jail_example_exec="/bin/sh /etc/rc"		# command to execute in jail
+#jail_example_exec_start="/bin/sh /etc/rc"		# command to execute in jail for starting
+#jail_example_exec_stop="/bin/sh /etc/rc.shutdown"	# command to execute in jail for stopping
 #jail_example_devfs_enable="NO"			# mount devfs in the jail
 #jail_example_fdescfs_enable="NO"		# mount fdescfs in the jail
 #jail_example_procfs_enable="NO"		# mount procfs in jail
diff --git a/etc/rc.d/jail b/etc/rc.d/jail
index 38d20c6..535a719 100644
--- a/etc/rc.d/jail
+++ b/etc/rc.d/jail
@@ -34,7 +34,21 @@ init_variables()
 	eval jail_hostname=\"\$jail_${_j}_hostname\"
 	eval jail_ip=\"\$jail_${_j}_ip\"
 	eval jail_exec=\"\$jail_${_j}_exec\"
-	[ -z "${jail_exec}" ] && jail_exec="/bin/sh /etc/rc"
+	eval jail_exec_start=\"\$jail_${_j}_exec_start\"
+	eval jail_exec_stop=\"\$jail_${_j}_exec_stop\"
+	if [ -n "${jail_exec}" ]; then
+		#   simple/backward-compatible execution
+		jail_exec_start="${jail_exec}"
+		jail_exec_stop=""
+	else
+		#   flexible execution
+		if [ -z "${jail_exec_start}" ]; then
+			jail_exec_start="/bin/sh /etc/rc"
+			if [ -z "${jail_exec_stop}" ]; then
+				jail_exec_stop="/bin/sh /etc/rc.shutdown"
+			fi
+		fi
+	fi
 
 	# The default jail ruleset will be used by rc.subr if none is specified.
 	eval jail_ruleset=\"\$jail_${_j}_devfs_ruleset\"
@@ -65,6 +79,8 @@ init_variables()
 	debug "$_j procdir: $jail_procdir"
 	debug "$_j ruleset: $jail_ruleset"
 	debug "$_j fstab: $jail_fstab"
+	debug "$_j exec start: $jail_exec_start"
+	debug "$_j exec stop: $jail_exec_stop"
 }
 
 # set_sysctl rc_knob mib msg
@@ -177,8 +193,8 @@ jail_start()
 			fi
 		fi
 		_tmp_jail=${_tmp_dir}/jail.$$
-		jail -i ${jail_rootdir} ${jail_hostname} \
-			${jail_ip} ${jail_exec} > ${_tmp_jail} 2>&1
+		eval jail -l -U root -i ${jail_rootdir} ${jail_hostname} \
+			${jail_ip} ${jail_exec_start} > ${_tmp_jail} 2>&1
 		[ "$?" -eq 0 ] && echo -n " $jail_hostname"
 		_jail_id=$(head -1 ${_tmp_jail})
 		tail +2 ${_tmp_jail} >${jail_rootdir}/var/log/console.log
@@ -198,7 +214,13 @@ jail_stop()
 			_jail_id=$(cat /var/run/jail_${_jail}.id)
 			if [ ! -z "${_jail_id}" ]; then
 				init_variables $_jail
+				if [ -n "${jail_exec_stop}" ]; then
+					eval env -i /usr/sbin/jexec ${_jail_id} ${jail_exec_stop} \
+						>> ${jail_rootdir}/var/log/console.log 2>&1
+				fi
 				killall -j ${_jail_id} -TERM > /dev/null 2>&1
+				sleep 1
+				killall -j ${_jail_id} -KILL > /dev/null 2>&1
 				jail_umount_fs
 				echo -n " $jail_hostname"
 			fi
diff --git a/etc/rc.shutdown b/etc/rc.shutdown
index 1d15154..075881f 100644
--- a/etc/rc.shutdown
+++ b/etc/rc.shutdown
@@ -80,7 +80,9 @@ fi
 # Determine the shutdown order of the /etc/rc.d scripts,
 # and perform the operation
 #
-files=`rcorder -k shutdown /etc/rc.d/* 2>/dev/null`
+rcorder_opts="-k shutdown"
+[ `/sbin/sysctl -n security.jail.jailed` -eq 1 ] && rcorder_opts="$rcorder_opts -s nojail"
+files=`rcorder ${rcorder_opts} /etc/rc.d/* 2>/dev/null`
 
 for _rc_elem in `reverse_list $files`; do
 	debug "run_rc_script $_rc_elem faststop"