Add a new package splitting mechanism to replace print-cdrom-packages.sh.

The new system tries to be more automated so that there is less work for
the re's to do.  It also no longer uses a /usr/ports tree as its input,
but uses the generated package build including its INDEX file as its input.
It parses the INDEX file, determines which packages should go on which ISO
images, and then builds full-fledged trees of packages that can be added
as an argument to mkisofs along with the tree built by 'make release' to
build a full CD image.  The INDEX files in the populated trees are
generated with volume media number to make use of sysinstall's multiple
volume support so that the user is kindly prompted to insert the
appropriate disc for a package if it is not on the current disc.  There is
still some more tweaking to be done here, but this part needs to be
committed.  This stuff will all be used to prep the 5.4 release as well.

Tested by:	kensmith, others on re@
Reviewed by:	re

2 files changed, 253 insertions, 0 deletions

diff --git a/release/scripts/package-split.py b/release/scripts/package-split.py
new file mode 100644
index 0000000..5c729c8
--- /dev/null
+++ b/release/scripts/package-split.py
@@ -0,0 +1,195 @@
+#!/usr/local/bin/python
+#
+# This script generates a master INDEX file for the CD images built by the
+# FreeBSD release engineers.  Each disc is given a list of desired packages.
+# Dependencies of these packages are placed on either the same disc or an
+# earlier disc.  The resulting master INDEX file is then written out.
+#
+# Usage: package-split.py <INDEX> <master INDEX>
+#
+# $FreeBSD$
+
+import os
+import sys
+
+try:
+    arch = os.environ["PKG_ARCH"]
+except:
+    arch = os.uname()[4]
+print "Using arch %s" % (arch)
+
+if 'PKG_VERBOSE' in os.environ:
+    verbose = 1
+else:
+    verbose = 0
+
+# List of packages for disc1.  This just includes packages sysinstall can
+# install as a distribution
+def disc1_packages():
+    # 5.x only
+    pkgs = ['lang/perl5.8']
+    if arch != 'ia64':
+        pkgs.append('x11/xorg')
+    if arch == 'alpha':
+        pkgs.append('emulators/osf1_base')
+    elif arch == 'i386':
+        pkgs.append('emulators/linux_base-8')
+    # 5.x only
+    if arch == 'i386':
+        pkgs.append('misc/compat22')
+        pkgs.append('misc/compat3x')
+        pkgs.append('misc/compat4x')
+    return pkgs
+
+# List of packages for disc2.  This includes packages that the X desktop
+# menu depends on (if it still exists) and other "nice to have" packages.
+# For architectures that use a separate livefs, this is actually disc3.
+def disc2_packages():
+            # X Desktops
+    pkgs = ['x11/kde3',
+            'x11/gnome2',
+            'x11-wm/afterstep',
+            'x11-wm/windowmaker',
+            'x11-wm/fvwm2',
+            # "Nice to have"
+            'archivers/unzip',
+            'astro/xearth',                 
+            'devel/gmake',
+            'editors/emacs',
+            'editors/vim',
+            'editors/vim-lite',
+            'editors/xemacs',
+            'emulators/mtools',
+            'graphics/png',
+            'graphics/xv',
+            'irc/xchat2',
+            'mail/exim',
+            'mail/fetchmail',
+            'mail/mutt',
+            'mail/pine4',
+            'mail/popd',
+            'mail/xfmail',
+            'mail/postfix',
+            'misc/bsdiff',
+            'net/cvsup',
+            'net/cvsup-without-gui',
+            'net/rsync',
+            'net/samba',
+            'news/slrn',
+            'news/tin',
+            'print/a2ps-letter',
+            'print/apsfilter',
+            'print/ghostscript-gnu-nox11',
+            'print/gv',
+            'print/psutils-letter',
+            'shells/bash2',
+            'shells/pdksh',
+            'shells/zsh',
+            'security/sudo',
+            'sysutils/portsnap',
+            'sysutils/portupgrade',
+            'www/links',
+            'www/lynx',
+            'x11/rxvt',
+            # Formerly on disc3
+            'lang/gnat',
+            'lang/php4',
+            'lang/php5',
+            'print/teTeX',
+            'security/portaudit',
+            'textproc/docproj-jadetex',
+            'www/apache13',
+            'www/apache13-modssl',
+            'www/apache2']
+    if arch == 'i386':
+        pkgs.append('comms/ltmdm')
+        pkgs.append('print/acroread')
+        pkgs.append('www/opera')
+    return pkgs
+
+# The list of desired packages
+def desired_packages():
+    disc1 = disc1_packages()
+    disc2 = disc2_packages()
+    return [disc1, disc2]
+
+# Suck the entire INDEX file into a two different dictionaries.  The first
+# dictionary maps port names (origins) to package names.  The second
+# dictionary maps a package name to a list of its dependent packages.
+PACKAGE_COL=0
+ORIGIN_COL=1
+DEPENDS_COL=8
+
+def load_index(index):
+    deps = {}
+    pkgs = {}
+    line_num = 1
+    for line in index:
+        fields = line.split('|')
+        name = fields[PACKAGE_COL]
+        if name in deps:
+            sys.stderr.write('%d: Duplicate package %s\n' % (line_num, name))
+            sys.exit(1)
+        origin = fields[ORIGIN_COL].replace('/usr/ports/', '', 1)
+        if origin in pkgs:
+            sys.stderr.write('%d: Duplicate port %s\n' % (line_num, origin))
+            sys.exit(1)
+        deps[name] = fields[DEPENDS_COL].split()
+        pkgs[origin] = name
+        line_num = line_num + 1
+    return (deps, pkgs)
+
+# Layout the packages on the various CD images.  Here's how it works.  We walk
+# each disc in the list of discs.  Within each disc we walk the list of ports.
+# For each port, we add the package name to a dictionary with the value being
+# the current disc number.  We also add all of the dependent packages.  If
+# a package is already in the dictionary when we go to add it, we just leave
+# the dictionary as it is.  This means that each package ends up on the first
+# disc that either lists it or contains it as a dependency.
+def layout_discs(discs, pkgs, deps):
+    disc_num = 1
+    layout = {}
+    for disc in discs:
+        for port in disc:
+            if port not in pkgs:
+                sys.stderr.write('Disc %d: Unable to find package for %s\n' %
+                                 (disc_num, port))
+                continue
+            pkg = pkgs[port]
+            pkg_list = [pkg] + deps[pkg]
+            for pkg in pkg_list:
+                if pkg not in layout:
+                    if verbose:
+                        print "--> Adding %s to Disc %d" % (pkg, disc_num)
+                    layout[pkg] = disc_num
+        disc_num = disc_num + 1
+    return layout
+
+# Generate a master INDEX file based on the generated layout.  The way this
+# works is that for each INDEX line, we check to see if the package is in the
+# layout.  If it is, we put that INDEX line into the master INDEX and append
+# a new field with the disc number to the line.
+def generate_index(index, layout, master_index):
+    for line in index:
+        pkg = line.split('|')[PACKAGE_COL]
+        if pkg in layout:
+            new_line = '%s|%d\n' % (line.splitlines()[0], layout[pkg])
+            master_index.write(new_line)
+
+# Verify the command line arguments
+if len(sys.argv) != 3:
+    sys.stderr.write('Invalid number of arguments\n')
+    sys.stderr.write('Usage: package-split.py <source INDEX> <master INDEX>\n')
+    sys.exit(1)
+
+print "Loading %s..." % (sys.argv[1])
+index = file(sys.argv[1])
+(deps, pkgs) = load_index(index)
+discs = desired_packages()
+layout = layout_discs(discs, pkgs, deps)
+index.seek(0)
+print "Generating %s..." % (sys.argv[2])
+master_index = file(sys.argv[2], 'w')
+generate_index(index, layout, master_index)
+index.close()
+master_index.close()
diff --git a/release/scripts/package-trees.sh b/release/scripts/package-trees.sh
new file mode 100644
index 0000000..7dc8ec5
--- /dev/null
+++ b/release/scripts/package-trees.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+#
+# This script generates the disk layout for the CD images built by the FreeBSD
+# release engineers as dictated by a specified master INDEX file.  Each disc
+# contains the master INDEX, it's assigned list of packages, and the
+# appropriate tree of category symlinks.
+#
+# Usage: package-tress.sh <copy method> <INDEX> <package tree> <destination>
+#
+# $FreeBSD$
+
+# Verify the command line
+if [ $# -ne 4 ]; then
+    echo "Invalid number of arguments"
+    echo "Usage: package-trees.sh <copy method> <INDEX> <tree> <destination>"
+    exit 1
+fi
+
+COPY=$1 ; shift
+INDEX=$1 ; shift
+TREE=$1 ; shift
+DESTDIR=$1 ; shift
+
+# First, determine the highest disc number.
+high_disc=`cut -d '|' -f 14 ${INDEX} | sort -n | tail -1`
+echo "Generating trees for ${high_disc} discs"
+
+# Second, initialize the trees for each disc
+for disc in `jot $high_disc`; do
+    rm -rf ${DESTDIR}/disc${disc}
+    mkdir -p ${DESTDIR}/disc${disc}/packages/All
+    cp ${INDEX} ${DESTDIR}/disc${disc}/packages/INDEX
+done
+
+# Third, run through the INDEX copying each package to its appropriate CD and
+# making the appropriate category symlinks
+while read line; do
+    disc=`echo $line | cut -d '|' -f 14`
+    package=`echo $line | cut -d '|' -f 1`
+    categories=`echo $line | cut -d '|' -f 7`
+    discdir=${DESTDIR}/disc${disc}
+    if [ -n "$PKG_VERBOSE" ]; then
+	echo "--> Copying $package to Disc $disc"
+    fi
+    ${COPY} ${TREE}/All/${package}.tbz ${discdir}/packages/All
+    for cat in ${categories}; do
+        catdir=${discdir}/packages/${cat}
+	mkdir -p ${catdir}
+	ln -s ../All/${package}.tbz ${catdir}
+    done
+done < ${INDEX}
+
+# Fourth, output du info for the relative size of the trees.
+discs=""
+for disc in `jot $high_disc`; do
+    discs="${discs} disc${disc}"
+done
+(cd ${DESTDIR}; du -sh ${discs})