Rewrite tbmaster to use configuration files instead of a hard-coded hash.

2 files changed, 283 insertions, 253 deletions

diff --git a/tools/tools/tinderbox/tbmaster.1 b/tools/tools/tinderbox/tbmaster.1
index 2ceba49..5256f28 100644
--- a/tools/tools/tinderbox/tbmaster.1
+++ b/tools/tools/tinderbox/tbmaster.1
@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 16, 2003
+.Dd July 23, 2003
 .Dt TBMASTER 1
 .Os
 .Sh NAME
@@ -35,99 +35,173 @@
 .Nd manage tinderbox runs
 .Sh SYNOPSIS
 .Nm
-.Oo
-.Ar setup
-.Oo
-.Ar arch Ns Op Ns : Ns Ar mach
-.Ar ...
-.Oc
-.Oc
+.Op options
 .Sh DESCRIPTION
 The
 .Nm
 script manages
 .Xr tinderbox 1
 runs, generates log summaries, and mails out failure reports.
-The first argument is the name of the setup to run (see
-.Sx Configuration
-below).
-Further arguments are target architectures or architecture / machine
-pairs to build for.
-If no arguments are specified, the setup corresponding to the hostname
-minus the domain part is run in full.
+.Pp
+The following options are recognized:
+.Bl -tag -width 12n
+.It Fl c Ar CONFIG , Fl -config Ns = Ns Ar CONFIG
+The name of the configuration to use.
+The default is the hostname minus the domain part.
+.It Fl e Ar DIR , Fl -etcdir Ns = Ns Ar DIR
+The directory where configuration files are located.
+The default is
+.Pa $HOME/etc .
+.El
 .Ss Configuration
-The configuration, which is contained in the script itself in the form
-of a Perl hash, consists of a list of setups which by convention are
-named for the hosts that they run on.
+The
+.Nm
+script uses named configurations located in individual files named for
+the setup they describe, with a
+.Pa .rc
+suffix.
+For instance, the
+.Dq snoosnoo
+configuration is contained in
+.Pa snoosnoo.rc .
+.Pp
+In addition,
+.Nm
+attempts to read
+.Pa default.rc
+before reading the actual configuration file; thus, that file may be
+used to specify default values shared by multiple configurations.
+.Pp
+The configuration consists of a list of single- or multiple-value
+variable assignments:
+.Bl -tag
+.It Va variable Ns = Ns Ar value
+.It Va variable Ns = Ns Ar value1 , Ar value 2, ...
+.El
 .Pp
-Each setup defines one or more of the following fields:
+Whitespace around the equal sign and around the commas separating
+multiple values is optional.
+.Pp
+Blank lines are ignoed, as is anything following a hash sign
+.Pq Dq # .
+.Pp
+The following configuration variables are defined:
 .Bl -tag -width 12n
-.It COMMENT
-.Pq Vt scalar
-A terse comment describing the setup.
+.It ARCH
+.Pq Vt simple
+The architecture currently being built for.
+Read-only.
+.It BRANCH
+.Pq Vt simple
+The branch currently being built.
+Read-only.
 .It BRANCHES
-.Pq Vt array
+.Pq Vt multiple
 A list of source branches to build.
-.It ARCHES
-.Pq Vt hash
-Which architectures and machines to build for.
-Each hash entry maps an architecture to a list of machines.
+The default value is
+.Dq CURRENT .
+.It COMMENT
+.Pq Vt single
+A terse comment describing the setup.
+No default value.
 .It DATE
-.Pq Vt scalar
+.Pq Vt single
 The date to use when checking out sources, or
 .Dq today
 for today's sources.
 If left undefined, the top of each branch is used.
-.It SANDBOX
-.Pq Vt scalar
-The location of the sandbox directory.
+No default value.
+.It ENV
+.Pq Vt multiple
+A list of environment variables to pass to the
+.Xr tinderbox 1
+script.
+Each value is the name and value of an environment variable, separated
+by an equal sign
+.Pq Dq = .
+No default value.
+.It HOME
+.Pq Vt single
+The current user's home directory, as specified by the
+.Ev HOME
+environment variable.
+Note that it will not be defined unless it passes some simple sanity
+checks.
+Read-only.
 .It LOGDIR
-.Pq Vt scalar
+.Pq Vt single
 The location of the log directory.
-.It TARGETS
-.Pq Vt array
-A list of targets (commands) to specify to the tinderbox script.
+The default value is
+.Pa %%SANDBOX%%/logs .
+.It MACHINE
+.Pq Vt simple
+The machine currently being built for.
+Read-only.
 .It OPTIONS
-.Pq Vt array
-A list of additional options to pass to the tinderbox script.
-.It ENV
-.Pq Vt hash
-Additional environment variables to pass to the tinderbox script.
-Each hash entry maps a variable's name to its value.
+.Pq Vt multiple
+A list of additional options to pass to the
+.Xr tinderbox 1
+script.
+No default value.
+.It PLATFORMS
+.Pq Vt multiple
+Which architectures and machines to build for.
+Each value is the name of an architecture, optionally followed by a
+forward slash
+.Pq Dq /
+and a machine name.
+If the machine name is not specified, it is assumed to be identical to
+the architecture name.
+The default value is
+.Dq i386 .
+.It RECIPIENT
+.Pq Vt single
+The address to which failure reports should be mailed.
+No default value.
+.It SANDBOX
+.Pq Vt single
+The location of the sandbox directory.
+The default value is
+.Pa /tmp/tinderbox .
 .It SENDER
-.Pq Vt scalar
+.Pq Vt single
 The envelope sender to use when mailing out failure reports.
 This should be a single email address.
-It is subject to variable substitution as described below.
-.It RECIPIENT
-.Pq Vt scalar
-The address to which failure reports should be mailed.
-It is subject to variable substitution as described below.
+No default value.
 .It SUBJECT
-.Pq Vt scalar
+.Pq Vt single
 The subject to use on failure reports.
-It is subject to variable substitution as described below.
+The default value is
+.Dq Tinderbox failure on %%arch%%/%%machine%% .
+.It TARGETS
+.Pq Vt multiple
+A list of targets (commands) to specify to the
+.Xr tinderbox 1
+script.
+The default is
+.Dq world .
+.It TINDERBOX
+The location of the
+.Xr tinderbox(1)
+script.
+The default value is
+.Dq %%HOME%%/bin/tinderbox .
 .El
 .Pp
 Fields which are left undefined are filled in from the
 .Dq global
 setup.
 .Ss Variable Substitution
-As noted above, certain configuration fields are subject to variable
-substitution.
-Recognized variable names include all the scalar configuration
-variables, plus
-.Va BRANCH ,
-.Va ARCH
-and
-.Va MACHINE
-which correspond to the source branch, target architecture and target
-machine of the failed build.
-.Pp
-Variables are specified by enclosing their name in double percentage
-signs.
-If the variable's name is spelled in lowercase, its value is converted
-to lowercase before substitution.
+All single-valued configuration variables are subject to variable
+substitution immediately before use.
+If a variable's value contains strings of the form
+.Va %%VAR%%
+or
+.Va %%var%% ,
+those strings are replaced with the values of the corresponding
+variables.
+The difference between the first and the second form is that the
+latter expands to lower-case.
 For instance,
 .Dq %%BRANCH%%
 might expand to
@@ -136,45 +210,6 @@ while
 .Dq %%branch%%
 would expand to
 .Dq releng_4 .
-.Sh EXAMPLES
-The following configuration includes two setups which build world,
-LINT and a release for
-.Fx
-Tier 1 and 2 platforms.
-.Bd -literal
-my %SETUPS      = (
-    'global' => {
-        'SANDBOX'       => '/home/bob/tinderbox',
-        'LOGDIR'        => '/home/bob/public_html',
-        'TARGETS'       => [ 'update', 'world', 'lint', 'release' ],
-        'OPTIONS'       => [ '--verbose' ],
-        'RECIPIENT'     => 'bob+%%arch%%-%%branch%%@example.com',
-        'SUBJECT'       => '[%%COMMENT%%] failure on %%arch%%/%%machine%%',
-        'ENV'           => {
-            'NOPROFILE' => 'YES',
-        },
-    },
-
-    'tier1' => {
-        'COMMENT'       => "Tier 1",
-        'BRANCHES'      => [ 'CURRENT' ],
-        'ARCHES'        => {
-            'alpha'     => [ 'alpha' ],
-            'i386       => [ 'i386', 'pc98' ],
-            'sparc64'   => [ 'sparc64' ],
-        },
-    },
-
-    'tier2' => {
-        'COMMENT'       => "Tier 2",
-        'BRANCHES'      => [ 'CURRENT' ],
-        'ARCHES'        => {
-            'ia64'      => [ 'ia64' ],
-            'powerpc'   => [ 'powerpc' ],
-        },
-    },
-);
-.Ed
 .Sh SEE ALSO
 .Xr perl 1 ,
 .Xr tinderbox 1
@@ -184,8 +219,3 @@ was written by
 .An Dag-Erling Smørgrav Aq des@FreeBSD.org .
 .Sh BUGS
 - part of a complete breakfast!
-.Pp
-The
-.Nm
-script should read its configuration from a file rather than have it
-hardcoded.
diff --git a/tools/tools/tinderbox/tbmaster.pl b/tools/tools/tinderbox/tbmaster.pl
index 190a78c..1916495 100644
--- a/tools/tools/tinderbox/tbmaster.pl
+++ b/tools/tools/tinderbox/tbmaster.pl
@@ -31,93 +31,93 @@
 
 use 5.006_001;
 use strict;
-use POSIX qw(tzset);
-use Sys::Hostname;
+use Fcntl qw(:DEFAULT :flock);
+use POSIX;
+use Getopt::Long;
 
-my %SETUPS	= (
-    # Global settings
-    'global' => {
-	'SANDBOX'	=> '/home/des/tinderbox',
-	'LOGDIR'	=> '/home/des/public_html',
-	'OPTIONS'	=> [ '--verbose' ],
-	'SENDER'	=> 'Tinderbox <des+tinderbox@freebsd.org>',
-	'RECIPIENT'	=> 'des+%%arch%%-%%branch%%@freebsd.org',
-	'SUBJECT'	=> '[%%COMMENT%%] failure on %%arch%%/%%machine%%',
-	'ENV'		=> { },
-    },
+my $VERSION	= "2.1";
+my $COPYRIGHT	= "Copyright (c) 2003 Dag-Erling Smørgrav. " .
+		  "All rights reserved.";
 
-    'cueball' => {
-	'COMMENT'	=> "-CURRENT tinderbox",
-	'BRANCHES'	=> [ 'CURRENT' ],
-	'TARGETS'	=> [ 'update', 'world', 'generic', 'lint' ],
-	'ARCHES'	=> {
-	    'alpha'	=> [ 'alpha' ],
-	    'amd64'	=> [ 'amd64' ],
-	    'i386'	=> [ 'i386', 'pc98' ],
-	    'ia64'	=> [ 'ia64' ],
-	    'sparc64'	=> [ 'sparc64' ],
-	},
-	'RECIPIENT'	=> 'current@freebsd.org,%%arch%%@freebsd.org',
-    },
+my $config;			# Name of current config
+my $etcdir;			# Configuration directory
 
-    'triangle' => {
-	'COMMENT'	=> "-STABLE tinderbox",
-	'BRANCHES'	=> [ 'RELENG_4' ],
-	'TARGETS'	=> [ 'update', 'world', 'generic', 'lint' ],
-	'ARCHES'	=> {
-	    'alpha'	=> [ 'alpha' ],
-	    'i386'	=> [ 'i386', 'pc98' ],
-	},
-	'ENV'		=> {
-	    'MAKE_KERBEROS5'	=> 'YES',
-	},
-	'RECIPIENT'	=> 'stable@freebsd.org,%%arch%%@freebsd.org',
-    },
+my %CONFIG = (
+    'COMMENT'	=> undef,
+    'BRANCHES'	=> [ 'CURRENT' ],
+    'PLATFORMS'	=> [ 'i386' ],
+    'DATE'	=> undef,
+    'SANDBOX'	=> '/tmp/tinderbox',
+    'LOGDIR'	=> '%%SANDBOX%%/logs',
+    'TARGETS'	=> [ 'update', 'world' ],
+    'OPTIONS'	=> [],
+    'ENV'	=> [],
+    'SENDER'	=> undef,
+    'RECIPIENT'	=> undef,
+    'SUBJECT'	=> 'Tinderbox failure on %%arch%%/%%machine%%',
+    'TINDERBOX'	=> '%%HOME%%/tinderbox',
+);
 
-    '9ball' => {
-	'COMMENT'	=> "Experimental platforms",
-	'BRANCHES'	=> [ 'CURRENT' ],
-	'TARGETS'	=> [ 'update', 'world', 'generic', 'lint' ],
-	'ARCHES'	=> {
-	    'powerpc'	=> [ 'powerpc' ],
-	},
-	'ENV'		=> {
-	    'NOLIBC_R'	=> 'YES',
-	    'NOFORTH'	=> 'YES',
-	},
-    },
+###
+### Perform variable expansion
+###
+sub expand($);
+sub expand($) {
+    my $key = shift;
 
-    'ada' => {
-	'COMMENT'	=> "Tinderbox development",
-	'BRANCHES'	=> [ 'RELENG_4' ],
-	'TARGETS'	=> [ 'update', 'world', 'lint', 'release' ],
-	'ARCHES'	=> {
-	    'i386'	=> [ 'i386' ],
-	},
-    },
+    return "??$key??"
+	unless exists($CONFIG{uc($key)});
+    return $CONFIG{uc($key)}
+	if (ref($CONFIG{uc($key)}));
+    my $str = $CONFIG{uc($key)};
+    while ($str =~ s/\%\%(\w+)\%\%/expand($1)/eg) {
+	# nothing
+    }
+    return ($key =~ m/[A-Z]/) ? $str : lc($str);
+}
 
-    'dsa' => {
-	'COMMENT'	=> "Tinderbox development",
-	'BRANCHES'	=> [ 'CURRENT' ],
-	'TARGETS'	=> [ 'update', 'world', 'lint', 'release' ],
-	'ARCHES'	=> {
-	    'alpha'	=> [ 'alpha' ],
-	},
-    },
+###
+### Read in a configuration file
+###
+sub readconf($) {
+    my $fn = shift;
 
-    'dwp' => {
-	'COMMENT'	=> "Tinderbox development",
-	'BRANCHES'	=> [ 'CURRENT' ],
-	'TARGETS'	=> [ 'update', 'world', 'lint', 'release' ],
-	'ARCHES'	=> {
-	    'i386'	=> [ 'i386' ],
-	},
-    },
-);
-my $config;			# Name of current config
-my %CONFIG = ();		# Current config
-my $TINDERBOX;			# Tinderbox script
+    local *CONF;
+    sysopen(CONF, $fn, O_RDONLY)
+	or return undef;
+    my $line = "";
+    my $n = 0;
+    while (<CONF>) {
+	++$n;
+	chomp();
+	s/\s*(\#.*)?$//;
+	$line .= $_;
+	if (length($line) && $line !~ s/\\$/ /) {
+	    die("$fn: syntax error on line $n\n")
+		unless ($line =~ m/^(\w+)\s*=\s*(.*)$/);
+	    my ($key, $val) = (uc($1), $2);
+	    die("$fn: unknown keyword on line $n\n")
+		unless (exists($CONFIG{$key}));
+	    if (ref($CONFIG{$key})) {
+		my @a = split(/\s*,\s*/, $val);
+		foreach (@a) {
+		    s/^\'([^\']*)\'$/$1/;
+		}
+		$CONFIG{$key} = \@a;
+	    } else {
+		$val =~ s/^\'([^\']*)\'$/$1/;
+		$CONFIG{$key} = $val;
+	    }
+	    $line = "";
+	}
+    }
+    close(CONF);
+    return 1;
+}
 
+###
+### Report a tinderbox failure
+###
 sub report($$$$) {
     my $sender = shift;
     my $recipient = shift;
@@ -140,18 +140,9 @@ sub report($$$$) {
     }
 }
 
-sub expand($) {
-    my $str = shift;
-
-    while (my ($key, $val) = each(%CONFIG)) {
-	next if ref($val);
-	$str =~ s/\%\%$key\%\%/$val/g;
-	($key, $val) = (lc($key), lc($val));
-	$str =~ s/\%\%$key\%\%/$val/g;
-    }
-    return $str;
-}
-
+###
+### Run the tinderbox
+###
 sub tinderbox($$$) {
     my $branch = shift;
     my $arch = shift;
@@ -162,7 +153,7 @@ sub tinderbox($$$) {
     $CONFIG{'MACHINE'} = $machine;
 
     # Open log files: one for the full log and one for the summary
-    my $logfile = "$CONFIG{'LOGDIR'}/tinderbox-$branch-$arch-$machine";
+    my $logfile = expand('LOGDIR') . "/tinderbox-$branch-$arch-$machine";
     local (*FULL, *BRIEF);
     if (!open(FULL, ">", "$logfile.full.$$")) {
 	warn("$logfile.full.$$: $!\n");
@@ -192,18 +183,16 @@ sub tinderbox($$$) {
 
     # Fork and start the tinderbox
     my @args = @{$CONFIG{'OPTIONS'}};
-    push(@args, "--sandbox=$CONFIG{'SANDBOX'}");
+    push(@args, "--sandbox=" . expand('SANDBOX'));
     push(@args, "--branch=$branch");
     push(@args, "--arch=$arch");
     push(@args, "--machine=$machine");
-    push(@args, "--date=$CONFIG{'DATE'}")
+    push(@args, "--date=" . expand('DATE'))
 	if (defined($CONFIG{'DATE'}));
-    push(@args, "--patch=$CONFIG{'PATCH'}")
+    push(@args, "--patch=" . expand('PATCH'))
 	if (defined($CONFIG{'PATCH'}));
     push(@args, @{$CONFIG{'TARGETS'}});
-    while (my ($key, $val) = each(%{$CONFIG{'ENV'}})) {
-	push(@args, "$key=$val");
-    }
+    push(@args, @{$CONFIG{'ENV'}});
     my $pid = fork();
     if (!defined($pid)) {
 	warn("fork(): $!\n");
@@ -217,7 +206,7 @@ sub tinderbox($$$) {
 	open(STDOUT, ">&WPIPE");
 	open(STDERR, ">&WPIPE");
 	$| = 1;
-	exec($TINDERBOX, @args);
+	exec(expand('TINDERBOX'), @args);
 	die("child: exec(): $!\n");
     }
 
@@ -273,10 +262,10 @@ sub tinderbox($$$) {
     close(FULL);
 
     # Mail out error reports
-    if ($error) {
-	my $sender = expand($CONFIG{'SENDER'});
-	my $recipient = expand($CONFIG{'RECIPIENT'});
-	my $subject = expand($CONFIG{'SUBJECT'});
+    if ($error && $CONFIG{'RECIPIENT'}) {
+	my $sender = expand('SENDER');
+	my $recipient = expand('RECIPIENT');
+	my $subject = expand('SUBJECT');
 	report($sender, $recipient, $subject, $summary);
     }
 
@@ -284,59 +273,70 @@ sub tinderbox($$$) {
     rename("$logfile.brief.$$", "$logfile.brief");
 }
 
+###
+### Print a usage message and exit
+###
 sub usage() {
 
-    my @configs = ();
-    foreach my $config (sort(keys(%SETUPS))) {
-	push(@configs, $config)
-	    unless ($config eq 'global');
-    }
-    print(STDERR "usage: tbmaster [", join('|', @configs), "]\n");
+    print(STDERR "This is the FreeBSD tinderbox manager, version $VERSION.
+$COPYRIGHT
+
+Usage:
+  $0 [options] [parameters]
+
+Parameters:
+  -c, --config=FILE             Configuration name
+  -e, --etcdir=DIR              Configuration directory
+
+Report bugs to <des\@freebsd.org>.
+");
+    print(STDERR "usage: tbmaster\n");
     exit(1);
 }
 
+###
+### Main
+###
 MAIN:{
-    if (@ARGV) {
-	$config = lc(shift(@ARGV));
-    } else {
-	$config = hostname();
-	$config =~ s/\..*//;
-    }
-    usage()
-	unless (exists($SETUPS{$config}) && $config ne 'global');
-    %CONFIG = %{$SETUPS{$config}};
-    foreach my $key (keys(%{$SETUPS{'global'}})) {
-	$CONFIG{$key} = $SETUPS{'global'}->{$key}
-	    unless (exists($CONFIG{$key}));
+    # Set defaults
+    $ENV{'PATH'} = "/usr/bin:/usr/sbin:/bin:/sbin";
+    $config = `uname -n`;
+    chomp($config);
+    $config =~ s/^(\w+)(\..*)?/$1/;
+    if ($ENV{'HOME'} =~ m|^((?:/[\w\.-]+)+)/*$|) {
+	$CONFIG{'HOME'} = $1;
+	$etcdir = "$1/etc";
+	$ENV{'PATH'} = "$1/bin:$ENV{'PATH'}"
+	    if (-d "$1/bin");
     }
 
-    $ENV{'TZ'} = "GMT";
-    tzset();
-    $ENV{'PATH'} = "";
-    $TINDERBOX = $0;
-    if ($TINDERBOX =~ m|(.*/)tbmaster(.*)$|) {
-	$TINDERBOX = "${1}tinderbox${2}";
+    # Get options
+    {Getopt::Long::Configure("auto_abbrev", "bundling");}
+    GetOptions(
+	"c|config=s"	        => \$config,
+	"e|etcdir=s"		=> \$etcdir,
+	) or usage();
+
+    if (defined($etcdir)) {
+	chdir($etcdir)
+	    or die("$etcdir: $!\n");
     }
-    if ($TINDERBOX eq $0 || ! -x $TINDERBOX) {
-	die("where is the tinderbox script?\n");
+    readconf('default.rc');
+    readconf("$config.rc")
+	or die("$config.rc: $!\n");
+    $CONFIG{'CONFIG'} = $config;
+    $CONFIG{'ETCDIR'} = $etcdir;
+
+    if (!length(expand('TINDERBOX')) || !-x expand('TINDERBOX')) {
+	die("Where is the tinderbox script?\n");
     }
 
-    foreach my $branch (sort(@{$CONFIG{'BRANCHES'}})) {
-	if (@ARGV) {
-	    foreach my $target (@ARGV) {
-		$target =~ m|^(\w+)(?:/(\w+))?$|
-		    or die("invalid target specification: $target\n");
-		my ($arch, $machine) = ($1, $2);
-		$machine = $arch
-		    unless defined($machine);
-		tinderbox($branch, $arch, $machine);
-	    }
-	} else {
-	    foreach my $arch (sort(keys(%{$CONFIG{'ARCHES'}}))) {
-		foreach my $machine (sort(@{$CONFIG{'ARCHES'}->{$arch}})) {
-		    tinderbox($branch, $arch, $machine);
-		}
-	    }
+    foreach my $branch (@{$CONFIG{'BRANCHES'}}) {
+	foreach my $platform (@{$CONFIG{'PLATFORMS'}}) {
+	    my ($arch, $machine) = split('/', $platform, 2);
+	    $machine = $arch
+		unless defined($machine);
+	    tinderbox($branch, $arch, $machine);
 	}
     }
 }