From e1798e6b444c47a33205b907ad7478a712c35ae5 Mon Sep 17 00:00:00 2001 From: cperciva Date: Mon, 12 Nov 2007 04:47:57 +0000 Subject: Add support for "freebsd-update -r newrelease upgrade" -- binary upgrading to new releases. Important parts of this code include * automatically determining which optional components (e.g., src, info, proflibs) are installed. * merging changes in files which are modified locally and have changed between the currently running and new release. * prompting the user to rebuild all 3rd party software before deleting old shared libraries. Yes, this is compatible with "freebsd-update rollback" -- you can test a new -BETA and roll back to the old release if you don't like it. Subject to re@ approval, this will be MFCed before 7.0-BETA3 and 6.3-RC1. MFC after: 2 days --- usr.sbin/freebsd-update/freebsd-update.8 | 16 +- usr.sbin/freebsd-update/freebsd-update.sh | 898 ++++++++++++++++++++++++++++-- 2 files changed, 850 insertions(+), 64 deletions(-) (limited to 'usr.sbin') diff --git a/usr.sbin/freebsd-update/freebsd-update.8 b/usr.sbin/freebsd-update/freebsd-update.8 index e934d82..f50d36a 100644 --- a/usr.sbin/freebsd-update/freebsd-update.8 +++ b/usr.sbin/freebsd-update/freebsd-update.8 @@ -1,5 +1,5 @@ .\"- -.\" Copyright 2006 Colin Percival +.\" Copyright 2006, 2007 Colin Percival .\" All rights reserved .\" .\" Redistribution and use in source and binary forms, with or without @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd August 30, 2006 +.Dd November 11, 2007 .Dt FREEBSD-UPDATE 8 .Os FreeBSD .Sh NAME @@ -37,6 +37,7 @@ .Op Fl d Ar workdir .Op Fl f Ar conffile .Op Fl k Ar KEY +.Op Fl r Ar newrelease .Op Fl s Ar server .Op Fl t Ar address .Cm command ... @@ -84,6 +85,10 @@ Read configuration options from Trust an RSA key with SHA256 of .Ar KEY . (default: read value from configuration file.) +.It Fl r Ar newrelease +Specify the new release to which +.Nm +should upgrade (upgrade command only). .It Fl s Ar server Fetch files from the specified server or server pool. (default: read value from configuration file.) @@ -118,8 +123,13 @@ from the random delay serves to minimize the probability that a large number of machines will simultaneously attempt to fetch updates. +.It Cm upgrade +Fetch files necessary for upgrading to a new release. +Before using this command, make sure that you read the +announcement and release notes for the new release in +case there are any special steps needed for upgrading. .It Cm install -Install the most recently fetched updates. +Install the most recently fetched updates or upgrade. .It Cm rollback Uninstall the most recently installed updates. .El diff --git a/usr.sbin/freebsd-update/freebsd-update.sh b/usr.sbin/freebsd-update/freebsd-update.sh index 6fd8bde..cccb0e2 100644 --- a/usr.sbin/freebsd-update/freebsd-update.sh +++ b/usr.sbin/freebsd-update/freebsd-update.sh @@ -44,6 +44,7 @@ Options: -f conffile -- Read configuration options from conffile (default: /etc/freebsd-update.conf) -k KEY -- Trust an RSA key with SHA256 hash of KEY + -r release -- Target for upgrade (e.g., 6.2-RELEASE) -s server -- Server from which to fetch updates (default: update.FreeBSD.org) -t address -- Mail output of cron command, if any, to address @@ -52,7 +53,8 @@ Commands: fetch -- Fetch updates from server cron -- Sleep rand(3600) seconds, fetch updates, and send an email if updates were found - install -- Install downloaded updates + upgrade -- Fetch upgrades to FreeBSD version specified via -r option + install -- Install downloaded updates or upgrades rollback -- Uninstall most recently installed updates EOF exit 0 @@ -84,7 +86,7 @@ EOF CONFIGOPTIONS="KEYPRINT WORKDIR SERVERNAME MAILTO ALLOWADD ALLOWDELETE KEEPMODIFIEDMETADATA COMPONENTS IGNOREPATHS UPDATEIFUNMODIFIED - BASEDIR VERBOSELEVEL" + BASEDIR VERBOSELEVEL TARGETRELEASE STRICTCOMPONENTS MERGECHANGES" # Set all the configuration options to "". nullconfig () { @@ -228,6 +230,14 @@ config_UpdateIfUnmodified () { done } +# Add to the list of paths within which updates to text files will be merged +# instead of overwritten. +config_MergeChanges () { + for C in $@; do + MERGECHANGES="${MERGECHANGES} ${C}" + done +} + # Work on a FreeBSD installation mounted under $1 config_BaseDir () { if [ -z ${BASEDIR} ]; then @@ -237,6 +247,36 @@ config_BaseDir () { fi } +# When fetching upgrades, should we assume the user wants exactly the +# components listed in COMPONENTS, rather than trying to guess based on +# what's currently installed? +config_StrictComponents () { + if [ -z ${STRICTCOMPONENTS} ]; then + case $1 in + [Yy][Ee][Ss]) + STRICTCOMPONENTS=yes + ;; + [Nn][Oo]) + STRICTCOMPONENTS=no + ;; + *) + return 1 + ;; + esac + else + return 1 + fi +} + +# Upgrade to FreeBSD $1 +config_TargetRelease () { + if [ -z ${TARGETRELEASE} ]; then + TARGETRELEASE=$1 + else + return 1 + fi +} + # Define what happens to output of utilities config_VerboseLevel () { if [ -z ${VERBOSELEVEL} ]; then @@ -313,6 +353,10 @@ parse_cmdline () { if [ $# -eq 1 ]; then usage; fi; shift config_ServerName $1 || usage ;; + -r) + if [ $# -eq 1 ]; then usage; fi; shift + config_TargetRelease $1 || usage + ;; -t) if [ $# -eq 1 ]; then usage; fi; shift config_MailTo $1 || usage @@ -331,7 +375,7 @@ parse_cmdline () { ;; # Commands - cron | fetch | install | rollback) + cron | fetch | upgrade | install | rollback) COMMANDS="${COMMANDS} $1" ;; @@ -407,6 +451,7 @@ default_params () { config_KeepModifiedMetadata yes config_BaseDir / config_VerboseLevel stats + config_StrictComponents no # Merge these defaults into the earlier-configured settings mergeconfig @@ -490,6 +535,7 @@ fetch_check_params () { sed -E 's,-SECURITY,-RELEASE,'` ARCH=`uname -m` FETCHDIR=${RELNUM}/${ARCH} + PATCHDIR=${RELNUM}/${ARCH}/bp # Figure out what directory contains the running kernel BOOTFILE=`sysctl -n kern.bootfile` @@ -529,6 +575,46 @@ fetch_check_params () { BDHASH=`echo ${BASEDIR} | sha256 -q` } +# Perform sanity checks etc. before fetching upgrades. +upgrade_check_params () { + fetch_check_params + + # Unless set otherwise, we're upgrading to the same kernel config. + NKERNCONF=${KERNCONF} + + # We need TARGETRELEASE set + _TARGETRELEASE_z="Release target must be specified via -r option." + if [ -z "${TARGETRELEASE}" ]; then + echo -n "`basename $0`: " + echo "${_TARGETRELEASE_z}" + exit 1 + fi + + # The target release should be != the current release. + if [ "${TARGETRELEASE}" = "${RELNUM}" ]; then + echo -n "`basename $0`: " + echo "Cannot upgrade from ${RELNUM} to itself" + exit 1 + fi + + # Turning off AllowAdd or AllowDelete is a bad idea for upgrades. + if [ "${ALLOWADD}" = "no" ]; then + echo -n "`basename $0`: " + echo -n "WARNING: \"AllowAdd no\" is a bad idea " + echo "when upgrading between releases." + echo + fi + if [ "${ALLOWDELETE}" = "no" ]; then + echo -n "`basename $0`: " + echo -n "WARNING: \"AllowDelete no\" is a bad idea " + echo "when upgrading between releases." + echo + fi + + # Set EDITOR to /usr/bin/vi if it isn't already set + : ${EDITOR:='/usr/bin/vi'} +} + # Perform sanity checks and set some final parameters in # preparation for installing updates. install_check_params () { @@ -749,6 +835,20 @@ fetch_progress () { echo -n " " } +# Function for asking the user if everything is ok +continuep () { + while read -p "Does this look reasonable (y/n)? " CONTINUE; do + case "${CONTINUE}" in + y*) + return 0 + ;; + n*) + return 1 + ;; + esac + done +} + # Initialize the working directory workdir_init () { mkdir -p files @@ -781,7 +881,8 @@ fetch_key () { # Fetch metadata signature, aka "tag". fetch_tag () { - echo ${NDEBUG} "Fetching metadata signature from ${SERVERNAME}... " + echo -n "Fetching metadata signature " + echo ${NDEBUG} "for ${RELNUM} from ${SERVERNAME}... " rm -f latest.ssl fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/latest.ssl \ 2>${QUIETREDIR} || true @@ -1072,23 +1173,34 @@ fetch_metadata () { return 0 } +# Extract a subset of a downloaded metadata file containing only the parts +# which are listed in COMPONENTS. +fetch_filter_metadata_components () { + METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'` + gunzip -c < files/${METAHASH}.gz > $1.all + + # Fish out the lines belonging to components we care about. + for C in ${COMPONENTS}; do + look "`echo ${C} | tr '/' '|'`|" $1.all + done > $1 + + # Remove temporary file. + rm $1.all +} + # Generate a filtered version of the metadata file $1 from the downloaded # file, by fishing out the lines corresponding to components we're trying # to keep updated, and then removing lines corresponding to paths we want # to ignore. fetch_filter_metadata () { - METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'` - gunzip -c < files/${METAHASH}.gz > $1.all - # Fish out the lines belonging to components we care about. + fetch_filter_metadata_components $1 + # Canonicalize directory names by removing any trailing / in # order to avoid listing directories multiple times if they # belong to multiple components. Turning "/" into "" doesn't # matter, since we add a leading "/" when we use paths later. - for C in ${COMPONENTS}; do - look "`echo ${C} | tr '/' '|'`|" $1.all - done | - cut -f 3- -d '|' | + cut -f 3- -d '|' $1 | sed -e 's,/|d|,|d|,' | sort -u > $1.tmp @@ -1100,24 +1212,23 @@ fetch_filter_metadata () { comm -13 - $1.tmp > $1 # Remove temporary files. - rm $1.all $1.tmp + rm $1.tmp } -# Filter the metadata file $1 by adding lines with "/boot/${KERNCONF}" +# Filter the metadata file $1 by adding lines with "/boot/$2" # replaced by ${KERNELDIR} (which is `sysctl -n kern.bootfile` minus the -# trailing "/kernel"); and if "/boot/${KERNCONF}" does not exist, remove +# trailing "/kernel"); and if "/boot/$2" does not exist, remove # the original lines which start with that. # Put another way: Deal with the fact that the FOO kernel is sometimes # installed in /boot/FOO/ and is sometimes installed elsewhere. fetch_filter_kernel_names () { - - grep ^/boot/${KERNCONF} $1 | - sed -e "s,/boot/${KERNCONF},${KERNELDIR},g" | + grep ^/boot/$2 $1 | + sed -e "s,/boot/$2,${KERNELDIR},g" | sort - $1 > $1.tmp mv $1.tmp $1 - if ! [ -d /boot/${KERNCONF} ]; then - grep -v ^/boot/${KERNCONF} $1 > $1.tmp + if ! [ -d /boot/$2 ]; then + grep -v ^/boot/$2 $1 > $1.tmp mv $1.tmp $1 fi } @@ -1200,10 +1311,43 @@ fetch_inspect_system () { echo "done." } +# For any paths matching ${MERGECHANGES}, compare $1 and $2 and find any +# files which differ; generate $3 containing these paths and the old hashes. +fetch_filter_mergechanges () { + # Pull out the paths and hashes of the files matching ${MERGECHANGES}. + for F in $1 $2; do + for X in ${MERGECHANGES}; do + grep -E "^${X}" ${F} + done | + cut -f 1,2,7 -d '|' | + sort > ${F}-values + done + + # Any line in $2-values which doesn't appear in $1-values and is a + # file means that we should list the path in $3. + comm -13 $1-values $2-values | + fgrep '|f|' | + cut -f 1 -d '|' > $2-paths + + # For each path, pull out one (and only one!) entry from $1-values. + # Note that we cannot distinguish which "old" version the user made + # changes to; but hopefully any changes which occur due to security + # updates will exist in both the "new" version and the version which + # the user has installed, so the merging will still work. + while read X; do + look "${X}|" $1-values | + head -1 + done < $2-paths > $3 + + # Clean up + rm $1-values $2-values $2-paths +} + # For any paths matching ${UPDATEIFUNMODIFIED}, remove lines from $[123] -# which correspond to lines in $2 with hashes not matching $1 or $3. For -# entries in $2 marked "not present" (aka. type -), remove lines from $[123] -# unless there is a corresponding entry in $1. +# which correspond to lines in $2 with hashes not matching $1 or $3, unless +# the paths are listed in $4. For entries in $2 marked "not present" +# (aka. type -), remove lines from $[123] unless there is a corresponding +# entry in $1. fetch_filter_unmodified_notpresent () { # Figure out which lines of $1 and $3 correspond to bits which # should only be updated if they haven't changed, and fish out @@ -1225,9 +1369,15 @@ fetch_filter_unmodified_notpresent () { sort > $2-values # Any entry in $2-values which is not in $1-values corresponds to - # a path which we need to remove from $1, $2, and $3. - comm -13 $1-values $2-values > mlines - rm $1-values $2-values + # a path which we need to remove from $1, $2, and $3, unless it + # that path appears in $4. + comm -13 $1-values $2-values | + sort -t '|' -k 1,1 > mlines.tmp + cut -f 1 -d '|' $4 | + sort | + join -v 2 -t '|' - mlines.tmp | + sort > mlines + rm $1-values $2-values mlines.tmp # Any lines in $2 which are not in $1 AND are "not present" lines # also belong in mlines. @@ -1346,6 +1496,51 @@ fetch_filter_uptodate () { mv $2.tmp $2 } +# Fetch any "clean" old versions of files we need for merging changes. +fetch_files_premerge () { + # We only need to do anything if $1 is non-empty. + if [ -s $1 ]; then + # Tell the user what we're doing + echo -n "Fetching files from ${OLDRELNUM} for merging... " + + # List of files wanted + fgrep '|f|' < $1 | + cut -f 3 -d '|' | + sort -u > files.wanted + + # Only fetch the files we don't already have + while read Y; do + if [ ! -f "files/${Y}.gz" ]; then + echo ${Y}; + fi + done < files.wanted > filelist + + # Actually fetch them + lam -s "${OLDFETCHDIR}/f/" - -s ".gz" < filelist | + xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ + 2>${QUIETREDIR} + + # Make sure we got them all, and move them into /files/ + while read Y; do + if ! [ -f ${Y}.gz ]; then + echo "failed." + return 1 + fi + if [ `gunzip -c < ${Y}.gz | + ${SHA256} -q` = ${Y} ]; then + mv ${Y}.gz files/${Y}.gz + else + echo "${Y} has incorrect hash." + return 1 + fi + done < filelist + echo "done." + + # Clean up + rm filelist files.wanted + fi +} + # Prepare to fetch files: Generate a list of the files we need, # copy the unmodified files we have into /files/, and generate # a list of patches to download. @@ -1421,7 +1616,7 @@ fetch_files () { echo -n "Fetching `wc -l < patchlist | tr -d ' '` " echo ${NDEBUG} "patches.${DDSTATS}" tr '|' '-' < patchlist | - lam -s "${FETCHDIR}/bp/" - | + lam -s "${PATCHDIR}/" - | xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 2>${STATSREDIR} | fetch_progress echo "done." @@ -1495,7 +1690,7 @@ fetch_create_manifest () { echo -n "been downloaded because the files have been " echo "modified locally:" cat modifiedfiles - fi + fi | more rm modifiedfiles # If no files will be updated, tell the user and exit @@ -1525,7 +1720,7 @@ fetch_create_manifest () { echo -n "The following files will be removed " echo "as part of updating to ${RELNUM}-p${RELPATCHNUM}:" cat files.removed - fi + fi | more rm files.removed # Report added files, if any @@ -1534,7 +1729,7 @@ fetch_create_manifest () { echo -n "The following files will be added " echo "as part of updating to ${RELNUM}-p${RELPATCHNUM}:" cat files.added - fi + fi | more rm files.added # Report updated files, if any @@ -1544,7 +1739,7 @@ fetch_create_manifest () { echo "as part of updating to ${RELNUM}-p${RELPATCHNUM}:" cat files.updated - fi + fi | more rm files.updated # Create a directory for the install manifest. @@ -1655,9 +1850,9 @@ fetch_run () { fetch_filter_metadata INDEX-NEW || return 1 fetch_filter_metadata INDEX-OLD || return 1 - # Translate /boot/`uname -i` into ${KERNELDIR} - fetch_filter_kernel_names INDEX-NEW - fetch_filter_kernel_names INDEX-OLD + # Translate /boot/${KERNCONF} into ${KERNELDIR} + fetch_filter_kernel_names INDEX-NEW ${KERNCONF} + fetch_filter_kernel_names INDEX-OLD ${KERNCONF} # For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the # system and generate an INDEX-PRESENT file. @@ -1668,7 +1863,8 @@ fetch_run () { # in INDEX-OLD or INDEX-NEW. Also remove lines where the entry in # INDEX-PRESENT has type - and there isn't a corresponding entry in # INDEX-OLD with type -. - fetch_filter_unmodified_notpresent INDEX-OLD INDEX-PRESENT INDEX-NEW + fetch_filter_unmodified_notpresent \ + INDEX-OLD INDEX-PRESENT INDEX-NEW /dev/null # For each entry in INDEX-PRESENT of type -, remove any corresponding # entry from INDEX-NEW if ${ALLOWADD} != "yes". Remove all entries @@ -1705,6 +1901,462 @@ fetch_run () { fetch_warn_eol || return 1 } +# If StrictComponents is not "yes", generate a new components list +# with only the components which appear to be installed. +upgrade_guess_components () { + if [ "${STRICTCOMPONENTS}" = "no" ]; then + # Generate filtered INDEX-ALL with only the components listed + # in COMPONENTS. + fetch_filter_metadata_components $1 || return 1 + + # Tell the user why his disk is suddenly making lots of noise + echo -n "Inspecting system... " + + # Look at the files on disk, and assume that a component is + # supposed to be present if it is more than half-present. + cut -f 1-3 -d '|' < INDEX-ALL | + tr '|' ' ' | + while read C S F; do + if [ -e ${BASEDIR}/${F} ]; then + echo "+ ${C}|${S}" + fi + echo "= ${C}|${S}" + done | + sort | + uniq -c | + sed -E 's,^ +,,' > compfreq + grep ' = ' compfreq | + cut -f 1,3 -d ' ' | + sort -k 2,2 -t ' ' > compfreq.total + grep ' + ' compfreq | + cut -f 1,3 -d ' ' | + sort -k 2,2 -t ' ' > compfreq.present + join -t ' ' -1 2 -2 2 compfreq.present compfreq.total | + while read S P T; do + if [ ${P} -gt `expr ${T} / 2` ]; then + echo ${S} + fi + done > comp.present + cut -f 2 -d ' ' < compfreq.total > comp.total + rm INDEX-ALL compfreq compfreq.total compfreq.present + + # We're done making noise. + echo "done." + + # Sometimes the kernel isn't installed where INDEX-ALL + # thinks that it should be: In particular, it is often in + # /boot/kernel instead of /boot/GENERIC or /boot/SMP. To + # deal with this, if "kernel|X" is listed in comp.total + # (i.e., is a component which would be upgraded if it is + # found to be present) we will add it to comp.present. + # If "kernel|" is in comp.total but "kernel|X" is + # not, we print a warning -- the user is running a kernel + # which isn't part of the release. + KCOMP=`echo ${KERNCONF} | tr 'A-Z' 'a-z'` + grep -E "^kernel\|${KCOMP}\$" comp.total >> comp.present + + if grep -qE "^kernel\|" comp.total && + ! grep -qE "^kernel\|${KCOMP}\$" comp.total; then + cat <<-EOF + +WARNING: This system is running a "${KCOMP}" kernel, which is not a +kernel configuration distributed as part of FreeBSD ${RELNUM}. +This kernel will not be updated: you MUST update the kernel manually +before running "$0 install". + EOF + fi + + # Re-sort the list of installed components and generate + # the list of non-installed components. + sort -u < comp.present > comp.present.tmp + mv comp.present.tmp comp.present + comm -13 comp.present comp.total > comp.absent + + # Ask the user to confirm that what we have is correct. To + # reduce user confusion, translate "X|Y" back to "X/Y" (as + # subcomponents must be listed in the configuration file). + echo + echo -n "The following components of FreeBSD " + echo "seem to be installed:" + tr '|' '/' < comp.present | + fmt -72 + echo + echo -n "The following components of FreeBSD " + echo "do not seem to be installed:" + tr '|' '/' < comp.absent | + fmt -72 + echo + continuep || return 1 + echo + + # Suck the generated list of components into ${COMPONENTS}. + # Note that comp.present.tmp is used due to issues with + # pipelines and setting variables. + COMPONENTS="" + tr '|' '/' < comp.present > comp.present.tmp + while read C; do + COMPONENTS="${COMPONENTS} ${C}" + done < comp.present.tmp + + # Delete temporary files + rm comp.present comp.present.tmp comp.absent comp.total + fi +} + +# If StrictComponents is not "yes", COMPONENTS contains an entry +# corresponding to the currently running kernel, and said kernel +# does not exist in the new release, add "kernel/generic" to the +# list of components. +upgrade_guess_new_kernel () { + if [ "${STRICTCOMPONENTS}" = "no" ]; then + # Grab the unfiltered metadata file. + METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'` + gunzip -c < files/${METAHASH}.gz > $1.all + + # If "kernel/${KCOMP}" is in ${COMPONENTS} and that component + # isn't in $1.all, we need to add kernel/generic. + for C in ${COMPONENTS}; do + if [ ${C} = "kernel/${KCOMP}" ] && + ! grep -qE "^kernel\|${KCOMP}\|" $1.all; then + COMPONENTS="${COMPONENTS} kernel/generic" + NKERNCONF="GENERIC" + cat <<-EOF + +WARNING: This system is running a "${KCOMP}" kernel, which is not a +kernel configuration distributed as part of FreeBSD ${RELNUM}. +As part of upgrading to FreeBSD ${RELNUM}, this kernel will be +replaced with a "generic" kernel. + EOF + continuep || return 1 + fi + done + + # Don't need this any more... + rm $1.all + fi +} + +# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into +# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades). +upgrade_oldall_to_oldnew () { + # For each ${F}|... which appears in INDEX-ALL but does not appear + # in INDEX-OLD, add ${F}|-|||||| to INDEX-OLD. + cut -f 1 -d '|' < $1 | + sort -u > $1.paths + cut -f 1 -d '|' < $2 | + sort -u | + comm -13 $1.paths - | + lam - -s "|-||||||" | + sort - $1 > $1.tmp + mv $1.tmp $1 + + # Remove lines from INDEX-OLD which also appear in INDEX-ALL + comm -23 $1 $2 > $1.tmp + mv $1.tmp $1 + + # Remove lines from INDEX-ALL which have a file name not appearing + # anywhere in INDEX-OLD (since these must be files which haven't + # changed -- if they were new, there would be an entry of type "-"). + cut -f 1 -d '|' < $1 | + sort -u > $1.paths + sort -k 1,1 -t '|' < $2 | + join -t '|' - $1.paths | + sort > $2.tmp + rm $1.paths + mv $2.tmp $2 + + # Rename INDEX-ALL to INDEX-NEW. + mv $2 $3 +} + +# From the list of "old" files in $1, merge changes in $2 with those in $3, +# and update $3 to reflect the hashes of merged files. +upgrade_merge () { + # We only need to do anything if $1 is non-empty. + if [ -s $1 ]; then + cut -f 1 -d '|' $1 | + sort > $1-paths + + # Create staging area for merging files + rm -rf merge/ + while read F; do + D=`dirname ${F}` + mkdir -p merge/old/${D} + mkdir -p merge/${OLDRELNUM}/${D} + mkdir -p merge/${RELNUM}/${D} + mkdir -p merge/new/${D} + done < $1-paths + + # Copy in files + while read F; do + # Currently installed file + V=`look "${F}|" $2 | cut -f 7 -d '|'` + gunzip < files/${V}.gz > merge/old/${F} + + # Old release + if look "${F}|" $1 | fgrep -q "|f|"; then + V=`look "${F}|" $1 | cut -f 3 -d '|'` + gunzip < files/${V}.gz \ + > merge/${OLDRELNUM}/${F} + fi + + # New release + if look "${F}|" $3 | cut -f 1,2,7 -d '|' | + fgrep -q "|f|"; then + V=`look "${F}|" $3 | cut -f 7 -d '|'` + gunzip < files/${V}.gz \ + > merge/${RELNUM}/${F} + fi + done < $1-paths + + # Attempt to automatically merge changes + echo -n "Attempting to automatically merge " + echo -n "changes in files..." + : > failed.merges + while read F; do + # If the file doesn't exist in the new release, + # the result of "merging changes" is having the file + # not exist. + if ! [ -f merge/${RELNUM}/${F} ]; then + continue + fi + + # If the file didn't exist in the old release, we're + # going to throw away the existing file and hope that + # the version from the new release is what we want. + if ! [ -f merge/${OLDRELNUM}/${F} ]; then + cp merge/${RELNUM}/${F} merge/new/${F} + continue + fi + + # Some files need special treatment. + case ${F} in + /etc/spwd.db | /etc/pwd.db | /etc/login.conf.db) + # Don't merge these -- we're rebuild them + # after updates are installed. + cp merge/old/${F} merge/new/${F} + ;; + *) + if ! merge -p -L "current version" \ + -L "${OLDRELNUM}" -L "${RELNUM}" \ + merge/old/${F} \ + merge/${OLDRELNUM}/${F} \ + merge/${RELNUM}/${F} \ + > merge/new/${F} 2>/dev/null; then + echo ${F} >> failed.merges + fi + ;; + esac + done < $1-paths + echo " done." + + # Ask the user to handle any files which didn't merge. + while read F; do + cat <<-EOF + +The following file could not be merged automatically: ${F} +Press Enter to edit this file in ${EDITOR} and resolve the conflicts +manually... + EOF + read dummy files/${V}.gz + echo "${F}|${V}" + fi + done < $1-paths > newhashes + + # Pull lines out from $3 which need to be updated to + # reflect merged files. + while read F; do + look "${F}|" $3 + done < $1-paths > $3-oldlines + + # Update lines to reflect merged files + join -t '|' -o 1.1,1.2,1.3,1.4,1.5,1.6,2.2,1.8 \ + $3-oldlines newhashes > $3-newlines + + # Remove old lines from $3 and add new lines. + sort $3-oldlines | + comm -13 - $3 | + sort - $3-newlines > $3.tmp + mv $3.tmp $3 + + # Clean up + rm $1-paths newhashes $3-oldlines $3-newlines + rm -rf merge/ + fi + + # We're done with merging files. + rm $1 +} + +# Do the work involved in fetching upgrades to a new release +upgrade_run () { + workdir_init || return 1 + + # Prepare the mirror list. + fetch_pick_server_init && fetch_pick_server + + # Try to fetch the public key until we run out of servers. + while ! fetch_key; do + fetch_pick_server || return 1 + done + + # Try to fetch the metadata index signature ("tag") until we run + # out of available servers; and sanity check the downloaded tag. + while ! fetch_tag; do + fetch_pick_server || return 1 + done + fetch_tagsanity || return 1 + + # Fetch the INDEX-OLD and INDEX-ALL. + fetch_metadata INDEX-OLD INDEX-ALL || return 1 + + # If StrictComponents is not "yes", generate a new components list + # with only the components which appear to be installed. + upgrade_guess_components INDEX-ALL || return 1 + + # Generate filtered INDEX-OLD and INDEX-ALL files containing only + # the components we want and without anything marked as "Ignore". + fetch_filter_metadata INDEX-OLD || return 1 + fetch_filter_metadata INDEX-ALL || return 1 + + # Merge the INDEX-OLD and INDEX-ALL files into INDEX-OLD. + sort INDEX-OLD INDEX-ALL > INDEX-OLD.tmp + mv INDEX-OLD.tmp INDEX-OLD + rm INDEX-ALL + + # Adjust variables for fetching files from the new release. + OLDRELNUM=${RELNUM} + RELNUM=${TARGETRELEASE} + OLDFETCHDIR=${FETCHDIR} + FETCHDIR=${RELNUM}/${ARCH} + + # Try to fetch the NEW metadata index signature ("tag") until we run + # out of available servers; and sanity check the downloaded tag. + while ! fetch_tag; do + fetch_pick_server || return 1 + done + + # Fetch the new INDEX-ALL. + fetch_metadata INDEX-ALL || return 1 + + # If StrictComponents is not "yes", COMPONENTS contains an entry + # corresponding to the currently running kernel, and said kernel + # does not exist in the new release, add "kernel/generic" to the + # list of components. + upgrade_guess_new_kernel INDEX-ALL || return 1 + + # Filter INDEX-ALL to contain only the components we want and without + # anything marked as "Ignore". + fetch_filter_metadata INDEX-ALL || return 1 + + # Convert INDEX-OLD (last release) and INDEX-ALL (new release) into + # INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades). + upgrade_oldall_to_oldnew INDEX-OLD INDEX-ALL INDEX-NEW + + # Translate /boot/${KERNCONF} or /boot/${NKERNCONF} into ${KERNELDIR} + fetch_filter_kernel_names INDEX-NEW ${NKERNCONF} + fetch_filter_kernel_names INDEX-OLD ${KERNCONF} + + # For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the + # system and generate an INDEX-PRESENT file. + fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1 + + # Based on ${MERGECHANGES}, generate a file tomerge-old with the + # paths and hashes of old versions of files to merge. + fetch_filter_mergechanges INDEX-OLD INDEX-PRESENT tomerge-old + + # Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which + # correspond to lines in INDEX-PRESENT with hashes not appearing + # in INDEX-OLD or INDEX-NEW. Also remove lines where the entry in + # INDEX-PRESENT has type - and there isn't a corresponding entry in + # INDEX-OLD with type -. + fetch_filter_unmodified_notpresent \ + INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old + + # For each entry in INDEX-PRESENT of type -, remove any corresponding + # entry from INDEX-NEW if ${ALLOWADD} != "yes". Remove all entries + # of type - from INDEX-PRESENT. + fetch_filter_allowadd INDEX-PRESENT INDEX-NEW + + # If ${ALLOWDELETE} != "yes", then remove any entries from + # INDEX-PRESENT which don't correspond to entries in INDEX-NEW. + fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW + + # If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in + # INDEX-PRESENT with metadata not matching any entry in INDEX-OLD, + # replace the corresponding line of INDEX-NEW with one having the + # same metadata as the entry in INDEX-PRESENT. + fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW + + # Remove lines from INDEX-PRESENT and INDEX-NEW which are identical; + # no need to update a file if it isn't changing. + fetch_filter_uptodate INDEX-PRESENT INDEX-NEW + + # Fetch "clean" files from the old release for merging changes. + fetch_files_premerge tomerge-old + + # Prepare to fetch files: Generate a list of the files we need, + # copy the unmodified files we have into /files/, and generate + # a list of patches to download. + fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1 + + # Fetch patches from to-${RELNUM}/${ARCH}/bp/ + PATCHDIR=to-${RELNUM}/${ARCH}/bp + fetch_files || return 1 + + # Merge configuration file changes. + upgrade_merge tomerge-old INDEX-PRESENT INDEX-NEW || return 1 + + # Create and populate install manifest directory; and report what + # updates are available. + fetch_create_manifest || return 1 + + # Leave a note behind to tell the "install" command that the kernel + # needs to be installed before the world. + touch ${BDHASH}-install/kernelfirst +} + # Make sure that all the file hashes mentioned in $@ have corresponding # gzipped files stored in /files/. install_verify () { @@ -1822,16 +2474,117 @@ install_delete () { rm newfiles killfiles } -# Update linker.hints if anything in /boot/ was touched -install_kldxref () { - if cat $@ | - grep -qE '^/boot/'; then - kldxref -R /boot/ +# Install new files, delete old files, and update linker.hints +install_files () { + # If we haven't already dealt with the kernel, deal with it. + if ! [ -f $1/kerneldone ]; then + grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD + grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW + + # Install new files + install_from_index INDEX-NEW || return 1 + + # Remove files which need to be deleted + install_delete INDEX-OLD INDEX-NEW || return 1 + + # Update linker.hints if necessary + if [ -s INDEX-OLD -o -s INDEX-NEW ]; then + kldxref -R /boot/ 2>/dev/null + fi + + # We've finished updating the kernel. + touch $1/kerneldone + + # Do we need to ask for a reboot now? + if [ -f $1/kernelfirst ] && + [ -s INDEX-OLD -o -s INDEX-NEW ]; then + cat <<-EOF + +Kernel updates have been installed. Please reboot and run +"$0 install" again to finish installing updates. + EOF + exit 0 + fi + fi + + # If we haven't already dealt with the world, deal with it. + if ! [ -f $1/worlddone ]; then + # Install new shared libraries next + grep -vE '^/boot/' $1/INDEX-NEW | + grep -E '/lib/.*\.so\.[0-9]+' > INDEX-NEW + install_from_index INDEX-NEW || return 1 + + # Deal with everything else + grep -vE '^/boot/' $1/INDEX-OLD | + grep -vE '/lib/.*\.so\.[0-9]+' > INDEX-OLD + grep -vE '^/boot/' $1/INDEX-NEW | + grep -vE '/lib/.*\.so\.[0-9]+' > INDEX-NEW + install_from_index INDEX-NEW || return 1 + install_delete INDEX-OLD INDEX-NEW || return 1 + + # Rebuild /etc/spwd.db and /etc/pwd.db if necessary. + if [ /etc/master.passwd -nt /etc/spwd.db ] || + [ /etc/master.passwd -nt /etc/pwd.db ]; then + pwd_mkdb /etc/master.passwd + fi + + # Rebuild /etc/login.conf.db if necessary. + if [ /etc/login.conf -nt /etc/login.conf.db ]; then + cap_mkdb /etc/login.conf + fi + + # We've finished installing the world and deleting old files + # which are not shared libraries. + touch $1/worlddone + + # Do we need to ask the user to portupgrade now? + grep -vE '^/boot/' $1/INDEX-NEW | + grep -E '/lib/.*\.so\.[0-9]+' | + cut -f 1 -d '|' | + sort > newfiles + if grep -vE '^/boot/' $1/INDEX-OLD | + grep -E '/lib/.*\.so\.[0-9]+' | + cut -f 1 -d '|' | + sort | + join -v 1 - newfiles | + grep -q .; then + cat <<-EOF + +Completing this upgrade requires removing old shared object files. +Please rebuild all installed 3rd party software (e.g., programs +installed from the ports tree) and then run "$0 install" +again to finish installing updates. + EOF + rm newfiles + exit 0 + fi + rm newfiles fi + + # Remove old shared libraries + grep -vE '^/boot/' $1/INDEX-NEW | + grep -E '/lib/.*\.so\.[0-9]+' > INDEX-NEW + grep -vE '^/boot/' $1/INDEX-OLD | + grep -E '/lib/.*\.so\.[0-9]+' > INDEX-OLD + install_delete INDEX-OLD INDEX-NEW || return 1 + + # Remove temporary files + rm INDEX-OLD INDEX-NEW } # Rearrange bits to allow the installed updates to be rolled back install_setup_rollback () { + # Remove the "reboot after installing kernel", "kernel updated", and + # "finished installing the world" flags if present -- they are + # irrelevant when rolling back updates. + if [ -f ${BDHASH}-install/kernelfirst ]; then + rm ${BDHASH}-install/kernelfirst + rm ${BDHASH}-install/kerneldone + fi + if [ -f ${BDHASH}-install/worlddone ]; then + rm ${BDHASH}-install/worlddone + fi + if [ -L ${BDHASH}-rollback ]; then mv ${BDHASH}-rollback ${BDHASH}-install/rollback fi @@ -1851,17 +2604,8 @@ install_run () { install_unschg ${BDHASH}-install/INDEX-OLD \ ${BDHASH}-install/INDEX-NEW || return 1 - # Install new files - install_from_index \ - ${BDHASH}-install/INDEX-NEW || return 1 - - # Remove files which we want to delete - install_delete ${BDHASH}-install/INDEX-OLD \ - ${BDHASH}-install/INDEX-NEW || return 1 - - # Update linker.hints if anything in /boot/ was touched - install_kldxref ${BDHASH}-install/INDEX-OLD \ - ${BDHASH}-install/INDEX-NEW + # Install new files, delete old files, and update linker.hints + install_files ${BDHASH}-install || return 1 # Rearrange bits to allow the installed updates to be rolled back install_setup_rollback @@ -1882,6 +2626,41 @@ rollback_setup_rollback () { fi } +# Install old files, delete new files, and update linker.hints +rollback_files () { + # Install old shared library files + grep -vE '^/boot/' $1/INDEX-OLD | + grep -E '/lib/.*\.so\.[0-9]+' > INDEX-OLD + install_from_index INDEX-OLD || return 1 + + # Deal with files which are neither kernel nor shared library + grep -vE '^/boot/' $1/INDEX-OLD | + grep -vE '/lib/.*\.so\.[0-9]+' > INDEX-OLD + grep -vE '^/boot/' $1/INDEX-NEW | + grep -vE '/lib/.*\.so\.[0-9]+' > INDEX-NEW + install_from_index INDEX-OLD || return 1 + install_delete INDEX-NEW INDEX-OLD || return 1 + + # Delete unneeded shared library files + grep -vE '^/boot/' $1/INDEX-OLD | + grep -E '/lib/.*\.so\.[0-9]+' > INDEX-OLD + grep -vE '^/boot/' $1/INDEX-NEW | + grep -E '/lib/.*\.so\.[0-9]+' > INDEX-NEW + install_delete INDEX-NEW INDEX-OLD || return 1 + + # Deal with kernel files + grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD + grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW + install_from_index INDEX-OLD || return 1 + install_delete INDEX-NEW INDEX-OLD || return 1 + if [ -s INDEX-OLD -o -s INDEX-NEW ]; then + kldxref -R /boot/ 2>/dev/null + fi + + # Remove temporary files + rm INDEX-OLD INDEX-NEW +} + # Actually rollback updates rollback_run () { echo -n "Uninstalling updates..." @@ -1901,17 +2680,8 @@ rollback_run () { install_unschg ${BDHASH}-rollback/INDEX-NEW \ ${BDHASH}-rollback/INDEX-OLD || return 1 - # Install new files - install_from_index \ - ${BDHASH}-rollback/INDEX-OLD || return 1 - - # Remove files which we want to delete - install_delete ${BDHASH}-rollback/INDEX-NEW \ - ${BDHASH}-rollback/INDEX-OLD || return 1 - - # Update linker.hints if anything in /boot/ was touched - install_kldxref ${BDHASH}-rollback/INDEX-NEW \ - ${BDHASH}-rollback/INDEX-OLD + # Install old files, delete new files, and update linker.hints + rollback_files ${BDHASH}-rollback || return 1 # Remove the rollback directory and the symlink pointing to it; and # rearrange bits to allow the previous set of updates to be rolled @@ -1963,6 +2733,12 @@ cmd_cron () { rm ${TMPFILE} } +# Fetch files for upgrading to a new release. +cmd_upgrade () { + upgrade_check_params + upgrade_run || exit 1 +} + # Install downloaded updates. cmd_install () { install_check_params -- cgit v1.1