MFC 258063,258064,258066,258097,258185,259134:

- Add an -s option to the regression tests that specifies a path to an
  alternate etcupdate.sh script to test.  This allows a non-installed
  version of the script to be tested more easily.
- Add a pre-world mode of updating similar to the -p option that can be
  passed to mergemaster.
- Fix a couple of issues with -F:
  - Fix ALWAYS_INSTALL to take precedence over the FreeBSD ID checks.
  - Fix the -F option in the case that the only upstream change is a
    change in the FreeBSD ID and the local file is removed.
- Refresh /etc/localtime after each update using tzsetup -r.
- Regenerate /var/db/services.db when /etc/services changes.

2 files changed, 219 insertions, 53 deletions

diff --git a/usr.sbin/etcupdate/etcupdate.8 b/usr.sbin/etcupdate/etcupdate.8
index a61f181..ffd0516 100644
--- a/usr.sbin/etcupdate/etcupdate.8
+++ b/usr.sbin/etcupdate/etcupdate.8
@@ -1,4 +1,4 @@
-.\" Copyright (c) 2010-2012 Advanced Computing Technologies LLC
+.\" Copyright (c) 2010-2013 Advanced Computing Technologies LLC
 .\" Written by: John H. Baldwin <jhb@FreeBSD.org>
 .\" All rights reserved.
 .\"
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 16, 2012
+.Dd December 9, 2013
 .Dt ETCUPDATE 8
 .Os
 .Sh NAME
@@ -33,7 +33,7 @@
 .Nd "manage updates to system files not updated by installworld"
 .Sh SYNOPSIS
 .Nm
-.Op Fl nBF
+.Op Fl npBF
 .Op Fl d Ar workdir
 .Op Fl r | Fl s Ar source | Fl t Ar tarball
 .Op Fl A Ar patterns
@@ -64,6 +64,7 @@
 .Op Fl M Ar options
 .Nm
 .Cm resolve
+.Op Fl p
 .Op Fl d Ar workdir
 .Op Fl D Ar destdir
 .Op Fl L Ar logfile
@@ -342,27 +343,7 @@ then nothing will be output.
 .Sh OPTIONS
 The following options are available.
 Note that most options do not apply to all modes.
-.Bl -tag -width ".Fl d Ar workdir"
-.It Fl B
-Do not build generated files in a private object tree.
-Instead,
-reuse the generated files from a previously built object tree that matches
-the source tree.
-This can be useful to avoid gratuitous conflicts in
-.Xr sendmail 8
-configuration
-files when bootstrapping.
-It can also be useful for building a tarball that matches a specific
-world build.
-.It Fl d Ar workdir
-Specify an alternate directory to use as the work directory.
-The work directory is used to store the
-.Dq current
-and
-.Dq previous
-trees as well as unresolved conflicts.
-The default work directory is
-.Pa <destdir>/var/db/etcupdate .
+.Bl -tag -width ".Fl A Ar patterns"
 .It Fl A Ar patterns
 Always install the new version of any files that match any of the patterns
 listed in
@@ -378,6 +359,17 @@ Note that ignored files specified via the
 variable or the
 .Fl I
 option will not be installed.
+.It Fl B
+Do not build generated files in a private object tree.
+Instead,
+reuse the generated files from a previously built object tree that matches
+the source tree.
+This can be useful to avoid gratuitous conflicts in
+.Xr sendmail 8
+configuration
+files when bootstrapping.
+It can also be useful for building a tarball that matches a specific
+world build.
 .It Fl D Ar destdir
 Specify an alternate destination directory as the target of a merge.
 This is analogous to the
@@ -388,6 +380,15 @@ The default destination directory is an empty string which results in
 merges updating
 .Pa /etc
 on the local machine.
+.It Fl d Ar workdir
+Specify an alternate directory to use as the work directory.
+The work directory is used to store the
+.Dq current
+and
+.Dq previous
+trees as well as unresolved conflicts.
+The default work directory is
+.Pa <destdir>/var/db/etcupdate .
 .It Fl F
 Ignore changes in the FreeBSD ID string when comparing files in the
 destination directory to files in either of the
@@ -481,6 +482,33 @@ option is not specified,
 then a temporary
 .Dq current
 tree will be extracted to perform the comparison.
+.It Fl p
+Enable
+.Dq pre-world
+mode.
+Only merge changes to files that are necessary to successfully run
+.Sq make installworld
+or
+.Sq make installkernel .
+When this flag is enabled,
+the existing
+.Dq current
+and
+.Dq previous
+trees are left alone.
+Instead,
+a temporary tree is populated with the necessary files.
+This temporary tree is compared against the
+.Dq current
+tree.
+This allows a normal update to be run after
+.Sq make installworld
+has completed.
+Any conflicts generated during a
+.Dq pre-world
+update should be resolved by a
+.Dq pre-world
+.Cm resolve .
 .It Fl r
 Do not update the
 .Dq current
@@ -667,6 +695,25 @@ has been removed from the
 tree,
 but it has been locally modified.
 The modified version of the file remains in the destination directory.
+.It "Needs update: /etc/localtime (required manual update via tzsetup(1))"
+The
+.Fa /var/db/zoneinfo
+file does not exist,
+so
+.Nm
+was not able to refresh
+.Fa /etc/localtime
+from its source file in
+.Fa /usr/share/zoneinfo .
+Running
+.Xr tzsetup 1
+will both refresh
+.Fa /etc/localtime
+and generate
+.Fa /var/db/zoneinfo
+permitting future updates to refresh
+.Fa /etc/localtime
+automatically.
 .It "Needs update: /etc/mail/aliases.db (required manual update via newaliases(1))"
 The file
 .Pa /etc/mail/aliases
diff --git a/usr.sbin/etcupdate/etcupdate.sh b/usr.sbin/etcupdate/etcupdate.sh
index 2216836..addcc14 100755
--- a/usr.sbin/etcupdate/etcupdate.sh
+++ b/usr.sbin/etcupdate/etcupdate.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright (c) 2010 Advanced Computing Technologies LLC
+# Copyright (c) 2010-2013 Advanced Computing Technologies LLC
 # Written by: John H. Baldwin <jhb@FreeBSD.org>
 # All rights reserved.
 #
@@ -61,14 +61,15 @@
 usage()
 {
 	cat <<EOF
-usage: etcupdate [-nBF] [-d workdir] [-r | -s source | -t tarball] [-A patterns]
-                 [-D destdir] [-I patterns] [-L logfile] [-M options]
+usage: etcupdate [-npBF] [-d workdir] [-r | -s source | -t tarball]
+                 [-A patterns] [-D destdir] [-I patterns] [-L logfile]
+                 [-M options]
        etcupdate build [-B] [-d workdir] [-s source] [-L logfile] [-M options]
                  <tarball>
        etcupdate diff [-d workdir] [-D destdir] [-I patterns] [-L logfile]
        etcupdate extract [-B] [-d workdir] [-s source | -t tarball] [-L logfile]
                  [-M options]
-       etcupdate resolve [-d workdir] [-D destdir] [-L logfile]
+       etcupdate resolve [-p] [-d workdir] [-D destdir] [-L logfile]
        etcupdate status [-d workdir] [-D destdir]
 EOF
 	exit 1
@@ -181,22 +182,31 @@ always_install()
 # $1 - directory to store new tree in
 build_tree()
 {
-	local make
+	local destdir dir file make
 
 	make="make $MAKE_OPTIONS"
 
 	log "Building tree at $1 with $make"
 	mkdir -p $1/usr/obj >&3 2>&1
-	(cd $SRCDIR; $make DESTDIR=$1 distrib-dirs) >&3 2>&1 || return 1
-
-	if ! [ -n "$nobuild" ]; then
-		(cd $SRCDIR; \
-	    MAKEOBJDIRPREFIX=$1/usr/obj $make _obj SUBDIR_OVERRIDE=etc &&
-	    MAKEOBJDIRPREFIX=$1/usr/obj $make everything SUBDIR_OVERRIDE=etc &&
-	    MAKEOBJDIRPREFIX=$1/usr/obj $make DESTDIR=$1 distribution) \
+	destdir=`realpath $1`
+
+	if [ -n "$preworld" ]; then
+		# Build a limited tree that only contains files that are
+		# crucial to installworld.
+		for file in $PREWORLD_FILES; do
+			dir=`dirname /$file`
+			mkdir -p $1/$dir >&3 2>&1 || return 1
+			cp -p $SRCDIR/$file $1/$file || return 1
+		done
+	elif ! [ -n "$nobuild" ]; then
+		(cd $SRCDIR; $make DESTDIR=$destdir distrib-dirs &&
+    MAKEOBJDIRPREFIX=$destdir/usr/obj $make _obj SUBDIR_OVERRIDE=etc &&
+    MAKEOBJDIRPREFIX=$destdir/usr/obj $make everything SUBDIR_OVERRIDE=etc &&
+    MAKEOBJDIRPREFIX=$destdir/usr/obj $make DESTDIR=$destdir distribution) \
 		    >&3 2>&1 || return 1
 	else
-		(cd $SRCDIR; $make DESTDIR=$1 distribution) >&3 2>&1 || return 1
+		(cd $SRCDIR; $make DESTDIR=$destdir distrib-dirs &&
+		    $make DESTDIR=$destdir distribution) >&3 2>&1 || return 1
 	fi
 	chflags -R noschg $1 >&3 2>&1 || return 1
 	rm -rf $1/usr/obj >&3 2>&1 || return 1
@@ -218,9 +228,15 @@ build_tree()
 # source tree.
 extract_tree()
 {
+	local files
+
 	# If we have a tarball, extract that into the new directory.
 	if [ -n "$tarball" ]; then
-		if ! (mkdir -p $NEWTREE && tar xf $tarball -C $NEWTREE) \
+		files=
+		if [ -n "$preworld" ]; then
+			files="$PREWORLD_FILES"
+		fi
+		if ! (mkdir -p $NEWTREE && tar xf $tarball -C $NEWTREE $files) \
 		    >&3 2>&1; then
 			echo "Failed to extract new tree."
 			remove_tree $NEWTREE
@@ -470,6 +486,39 @@ diffnode()
 	esac
 }
 
+# Run one-off commands after an update has completed.  These commands
+# are not tied to a specific file, so they cannot be handled by
+# post_install_file().
+post_update()
+{
+	local args
+
+	# None of these commands should be run for a pre-world update.
+	if [ -n "$preworld" ]; then
+		return
+	fi
+
+	# If /etc/localtime exists and is not a symlink and /var/db/zoneinfo
+	# exists, run tzsetup -r to refresh /etc/localtime.
+	if [ -f ${DESTDIR}/etc/localtime -a \
+	    ! -L ${DESTDIR}/etc/localtime ]; then
+		if [ -f ${DESTDIR}/var/db/zoneinfo ]; then
+			if [ -n "${DESTDIR}" ]; then
+				args="-C ${DESTDIR}"
+			else
+				args=""
+			fi
+			log "tzsetup -r ${args}"
+			if [ -z "$dryrun" ]; then
+				tzsetup -r ${args} >&3 2>&1
+			fi
+		else
+			warn "Needs update: /etc/localtime (required" \
+			    "manual update via tzsetup(1))"
+		fi
+	fi
+}
+
 # Create missing parent directories of a node in a target tree
 # preserving the owner, group, and permissions from a specified
 # template tree.
@@ -567,6 +616,14 @@ post_install_file()
 				fi
 			fi
 			;;
+		/etc/services)
+			log "services_mkdb -q -o $DESTDIR/var/db/services.db" \
+			    "${DESTDIR}$1"
+			if [ -z "$dryrun" ]; then
+				services_mkdb -q -o $DESTDIR/var/db/services.db \
+				    ${DESTDIR}$1 >&3 2>&1
+			fi
+			;;
 	esac
 }
 
@@ -1010,16 +1067,6 @@ handle_modified_file()
 		fi
 	fi
 
-	# If the only change in the new file versus the old file is a
-	# change in the FreeBSD ID string and -F is specified, just
-	# update the FreeBSD ID string in the local file.
-	if [ -n "$FREEBSD_ID" -a $cmp -eq $COMPARE_DIFFFILES ] && \
-	    fbsdid_only $OLDTREE/$file $NEWTREE/$file; then
-		if update_freebsdid $file; then
-			continue
-		fi
-	fi
-
 	# If the file was removed from the dest tree, just whine.
 	if [ $newdestcmp -eq $COMPARE_ONLYFIRST ]; then
 		# If the removed file matches an ALWAYS_INSTALL glob,
@@ -1034,6 +1081,14 @@ handle_modified_file()
 			return
 		fi
 
+		# If the only change in the new file versus the old
+		# file is a change in the FreeBSD ID string and -F is
+		# specified, don't warn.
+		if [ -n "$FREEBSD_ID" -a $cmp -eq $COMPARE_DIFFFILES ] && \
+		    fbsdid_only $OLDTREE/$file $NEWTREE/$file; then
+			return
+		fi
+
 		case $cmp in
 			$COMPARE_DIFFTYPE)
 				old=`file_type $OLDTREE/$file`
@@ -1064,6 +1119,16 @@ handle_modified_file()
 		fi
 	fi
 
+	# If the only change in the new file versus the old file is a
+	# change in the FreeBSD ID string and -F is specified, just
+	# update the FreeBSD ID string in the local file.
+	if [ -n "$FREEBSD_ID" -a $cmp -eq $COMPARE_DIFFFILES ] && \
+	    fbsdid_only $OLDTREE/$file $NEWTREE/$file; then
+		if update_freebsdid $file; then
+			continue
+		fi
+	fi
+
 	# If the file changed types between the old and new trees but
 	# the files in the new and dest tree are both of the same
 	# type, treat it like an added file just comparing the new and
@@ -1298,6 +1363,11 @@ resolve_cmd()
 		return
 	fi
 
+	if ! [ -d $NEWTREE ]; then
+		echo "The current tree is not present to resolve conflicts."
+		exit 1
+	fi
+
 	conflicts=`(cd $CONFLICTS; find . ! -type d) | sed -e 's/^\.//'`
 	for file in $conflicts; do
 		resolve_conflict $file
@@ -1343,7 +1413,7 @@ update_cmd()
 		usage
 	fi
 
-	log "update command: rerun=$rerun tarball=$tarball"
+	log "update command: rerun=$rerun tarball=$tarball preworld=$preworld"
 
 	if [ `id -u` -ne 0 ]; then
 		echo "Must be root to update a tree."
@@ -1376,9 +1446,22 @@ update_cmd()
 				echo "Unable to create temporary directory."
 				exit 1
 			fi
-			OLDTREE=$NEWTREE
+
+			# A pre-world dryrun has already set OLDTREE to
+			# point to the current stock tree.
+			if [ -z "$preworld" ]; then
+				OLDTREE=$NEWTREE
+			fi
 			NEWTREE=$dir
 
+		# For a pre-world update, blow away any pre-existing
+		# NEWTREE.
+		elif [ -n "$preworld" ]; then
+			if ! remove_tree $NEWTREE; then
+				echo "Unable to remove pre-world tree."
+				exit 1
+			fi
+
 		# Rotate the existing stock tree to the old tree.
 		elif [ -d $NEWTREE ]; then
 			# First, delete the previous old tree if it exists.
@@ -1421,6 +1504,12 @@ EOF
 	# Initialize conflicts and warnings handling.
 	rm -f $WARNINGS
 	mkdir -p $CONFLICTS
+
+	# Ignore removed files for the pre-world case.  A pre-world
+	# update uses a stripped-down tree.
+	if [ -n "$preworld" ]; then
+		> $WORKDIR/removed.files
+	fi
 	
 	# The order for the following sections is important.  In the
 	# odd case that a directory is converted into a file, the
@@ -1458,6 +1547,9 @@ EOF
 		    "(requires manual update via newaliases(1))"
 	fi
 
+	# Run any special one-off commands after an update has completed.
+	post_update
+
 	if [ -s $WARNINGS ]; then
 		echo "Warnings:"
 		cat $WARNINGS
@@ -1535,7 +1627,8 @@ always=
 dryrun=
 ignore=
 nobuild=
-while getopts "d:nrs:t:A:BD:FI:L:M:" option; do
+preworld=
+while getopts "d:nprs:t:A:BD:FI:L:M:" option; do
 	case "$option" in
 		d)
 			WORKDIR=$OPTARG
@@ -1543,6 +1636,9 @@ while getopts "d:nrs:t:A:BD:FI:L:M:" option; do
 		n)
 			dryrun=YES
 			;;
+		p)
+			preworld=YES
+			;;
 		r)
 			rerun=YES
 			;;
@@ -1633,6 +1729,9 @@ WARNINGS=$WORKDIR/warnings
 # Use $EDITOR for resolving conflicts.  If it is not set, default to vi.
 EDITOR=${EDITOR:-/usr/bin/vi}
 
+# Files that need to be updated before installworld.
+PREWORLD_FILES="etc/master.passwd etc/group"
+
 # Handle command-specific argument processing such as complaining
 # about unsupported options.  Since the configuration file is always
 # included, do not complain about extra command line arguments that
@@ -1644,19 +1743,39 @@ case $command in
 			echo
 			usage
 		fi
+		if [ -n "$rerun" -a -n "$preworld" ]; then
+			echo "Only one of -p or -r can be specified."
+			echo
+			usage
+		fi
+		;;
+	build|diff|status)
+		if [ -n "$dryrun" -o -n "$rerun" -o -n "$tarball" -o \
+		     -n "$preworld" ]; then
+			usage
+		fi
 		;;
-	build|diff|resolve|status)
+	resolve)
 		if [ -n "$dryrun" -o -n "$rerun" -o -n "$tarball" ]; then
 			usage
 		fi
 		;;
 	extract)
-		if [ -n "$dryrun" -o -n "$rerun" ]; then
+		if [ -n "$dryrun" -o -n "$rerun" -o -n "$preworld" ]; then
 			usage
 		fi
 		;;
 esac
 
+# Pre-world mode uses a different set of trees.  It leaves the current
+# tree as-is so it is still present for a full etcupdate run after the
+# world install is complete.  Instead, it installs a few critical files
+# into a separate tree.
+if [ -n "$preworld" ]; then
+	OLDTREE=$NEWTREE
+	NEWTREE=$WORKDIR/preworld
+fi
+
 # Open the log file.  Don't truncate it if doing a minor operation so
 # that a minor operation doesn't lose log info from a major operation.
 if ! mkdir -p $WORKDIR 2>/dev/null; then