Add a facility for a ``slow'' CTM delta queue, allowing `x' number

of delta's to be mailed out every hour (or however often you schedule
the cron job).

ctm_dequeue is the cron job which takes the stuff from the
queue directory and punts it into sendmail. The chunks of
the deltas (and the complete deltas if they are that small)
are sorted into order before being dispatched, so the people
subscribing should still get the bits in the right order.

The changes to ctm_smail should be fairly safe as they won't be
activated unless you go for the new queue directory option.

3 files changed, 497 insertions, 4 deletions

diff --git a/usr.sbin/ctm/ctm_dequeue/Makefile b/usr.sbin/ctm/ctm_dequeue/Makefile
new file mode 100644
index 0000000..543e77e
--- /dev/null
+++ b/usr.sbin/ctm/ctm_dequeue/Makefile
@@ -0,0 +1,8 @@
+PROG=           ctm_dequeue
+SRCS=           ctm_dequeue.c error.c
+NOMAN=          yes
+CFLAGS+=	-Wall -I${.CURDIR}/../ctm_rmail
+
+.PATH: ${.CURDIR}/../ctm_rmail
+
+.include <bsd.prog.mk>
diff --git a/usr.sbin/ctm/ctm_dequeue/ctm_dequeue.c b/usr.sbin/ctm/ctm_dequeue/ctm_dequeue.c
new file mode 100644
index 0000000..9d10fc5
--- /dev/null
+++ b/usr.sbin/ctm/ctm_dequeue/ctm_dequeue.c
@@ -0,0 +1,224 @@
+/*
+ * Copyright (c) 1996, Gary J. Palmer
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    verbatim and that no modifications are made prior to this
+ *    point in the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id$
+ */
+
+/* 
+ * Change this if you want to alter how many files it sends out by
+ * default
+ */
+
+#define DEFAULT_NUM 2
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fts.h>
+#include <sys/mman.h>
+#include <errno.h>
+#include <paths.h>
+#include "error.h"
+#include "options.h"
+
+int fts_sort(const FTSENT **, const FTSENT **);
+FILE *open_sendmail(void);
+int close_sendmail(FILE *fp);
+
+int
+main(int argc, char **argv)
+{
+    char *log_file = NULL;
+    char *queue_dir = NULL;
+    char *list[2];
+    char *buffer, *filename;
+    int num_to_send = DEFAULT_NUM, piece, fp, len;
+    FTS *fts;
+    FTSENT *ftsent;
+    FILE *sfp;
+
+    err_prog_name(argv[0]);
+
+    OPTIONS("[-l log] [-n num] queuedir")
+	NUMBER('n', num_to_send)
+	STRING('l', log_file)
+    ENDOPTS;
+
+    if (argc != 2)
+	usage();
+
+    queue_dir = argv[1];
+    list[0] = queue_dir;
+    list[1] = NULL;
+
+    fts = fts_open(list, FTS_PHYSICAL, fts_sort);
+    if (fts == NULL)
+    {
+	err("fts failed on `%s'", queue_dir);
+	exit(1);
+    }
+
+    (void) fts_read(fts);
+
+    ftsent = fts_children(fts, 0);
+    if (ftsent == NULL)
+    {
+	err("ftschildren failed");
+	exit(1);
+    }
+
+    /* assumption :-( */
+    len = strlen(queue_dir) + 40;
+    filename = malloc(len);
+    if (filename == NULL)
+    {
+	err("malloc failed");
+	exit(1);
+    }
+
+    for (piece = 0; piece < num_to_send ; piece++)
+    {
+	/* Skip non-files and files we should ignore (ones starting with `.') */
+
+#define ISFILE ((ftsent->fts_info & FTS_F) == FTS_F)
+#define IGNORE (ftsent->fts_name[0] == '.')
+#define HASNEXT (ftsent->fts_link != NULL)
+
+	while(((!ISFILE) || (IGNORE)) && (HASNEXT))
+	    ftsent = ftsent->fts_link;
+
+	if ((!ISFILE) || (IGNORE) || (!HASNEXT))
+	{
+	    err("No more chunks to mail");
+	    exit(0);
+	}
+
+#undef ISFILE
+#undef IGNORE
+#undef HASNEXT
+
+	if (snprintf(filename, len, "%s/%s", queue_dir, ftsent->fts_name) > len)
+	    err("snprintf(filename) longer than buffer");
+
+	fp = open(filename, O_RDONLY, 0);
+	if (fp <  0)
+	{
+	    err("open(`%s') failed, errno = %d", filename, errno);
+	    exit(1);
+	}
+
+	buffer = mmap(0, ftsent->fts_statp->st_size, PROT_READ, MAP_PRIVATE, fp, 0);
+	if (((int) buffer) <= 0)
+	{
+	    err("mmap failed, errno = %d", errno);
+	    exit(1);
+	}
+
+	sfp = open_sendmail();	    
+	if (sfp == NULL)
+	    exit(1);
+	
+	if (fwrite(buffer, ftsent->fts_statp->st_size, 1, sfp) < 1)
+	{
+	    err("fwrite failed: errno = %d", errno);
+	    close_sendmail(sfp);
+	    exit(1);
+	}
+
+	if (!close_sendmail(sfp))
+	    exit(1);
+
+	munmap(buffer, ftsent->fts_statp->st_size);
+	close(fp);
+	
+	err("sent file `%s'", ftsent->fts_name);
+
+	if (ftsent->fts_link != NULL)
+	    ftsent = ftsent->fts_link;
+    }
+    err("exiting normally");
+    return(0);
+}
+
+int
+fts_sort(const FTSENT ** a, const FTSENT ** b)
+{
+	int a_info, b_info;
+
+	a_info = (*a)->fts_info;
+	if (a_info == FTS_ERR)
+		return (0);
+	b_info = (*b)->fts_info;
+	if (b_info == FTS_ERR)
+		return (0);
+
+	return (strcmp((*a)->fts_name, (*b)->fts_name));
+}
+
+/*
+ * Start a pipe to sendmail.  Sendmail will decode the destination
+ * from the message contents.
+ */
+FILE *
+open_sendmail()
+{
+    FILE *fp;
+    char buf[100];
+    
+    sprintf(buf, "%s -t", _PATH_SENDMAIL);
+    if ((fp = popen(buf, "w")) == NULL)
+	err("cannot start sendmail");
+    return fp;
+}
+
+
+/*
+ * Close a pipe to sendmail.  Sendmail will then do its bit.
+ * Return 1 on success, 0 on failure.
+ */
+int
+close_sendmail(FILE *fp)
+{
+    int status;
+    
+    fflush(fp);
+    if (ferror(fp))
+    {
+	err("error writing to sendmail");
+	return 0;
+    }
+    
+    if ((status = pclose(fp)) != 0)
+	err("sendmail failed with status %d", status);
+    
+    return (status == 0);
+}
diff --git a/usr.sbin/ctm/ctm_smail/ctm_smail.c b/usr.sbin/ctm/ctm_smail/ctm_smail.c
index 1b2aaed..4c97e8a 100644
--- a/usr.sbin/ctm/ctm_smail/ctm_smail.c
+++ b/usr.sbin/ctm/ctm_smail/ctm_smail.c
@@ -9,11 +9,15 @@
  * NOTICE: This is free software.  I hope you get some use from this program.
  * In return you should think about all the nice people who give away software.
  * Maybe you should write some free software too.
+ *
+ * $Id$
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <errno.h>
@@ -27,6 +31,8 @@
 
 void chop_and_send(char *delta, off_t ctm_size, long max_msg_size,
 	char *mail_alias);
+void chop_and_queue(char *delta, off_t ctm_size, long max_msg_size,
+	char *queue_dir, char *mail_alias);
 unsigned encode_body(FILE *sm_fp, FILE *delta_fp, long msg_size);
 void write_header(FILE *sfp, char *mail_alias, char *delta, int pce,
 	int npieces);
@@ -35,7 +41,9 @@ void apologise(char *delta, off_t ctm_size, long max_ctm_size,
 	char *mail_alias);
 FILE *open_sendmail(void);
 int close_sendmail(FILE *fp);
-
+int lock_queuedir(char *queue_dir);
+void free_lock(int lockf, char *queue_dir);
+void add_to_queue(char *queue_dir, char *mail_alias, char *delta, int npieces, char **tempnames);
 
 int
 main(int argc, char **argv)
@@ -45,14 +53,16 @@ main(int argc, char **argv)
     long max_msg_size = DEF_MAX_MSG;
     long max_ctm_size = 0;
     char *log_file = NULL;
+    char *queue_dir = NULL;
     struct stat sb;
 
     err_prog_name(argv[0]);
 
-    OPTIONS("[-l log] [-m maxmsgsize] [-c maxctmsize] ctm-delta mail-alias")
+    OPTIONS("[-l log] [-m maxmsgsize] [-c maxctmsize] [-q queuedir] ctm-delta mail-alias")
 	NUMBER('m', max_msg_size)
 	NUMBER('c', max_ctm_size)
 	STRING('l', log_file)
+	STRING('q', queue_dir)
     ENDOPTS
 
     if (argc != 3)
@@ -72,8 +82,10 @@ main(int argc, char **argv)
 
     if (max_ctm_size != 0 && sb.st_size > max_ctm_size)
 	apologise(delta_file, sb.st_size, max_ctm_size, mail_alias);
-    else
+    else if (queue_dir == NULL)
 	chop_and_send(delta_file, sb.st_size, max_msg_size, mail_alias);
+    else
+	chop_and_queue(delta_file, sb.st_size, max_msg_size, queue_dir, mail_alias);
 
     return 0;
     }
@@ -93,6 +105,10 @@ chop_and_send(char *delta, off_t ctm_size, long max_msg_size, char *mail_alias)
     FILE *dfp;
     unsigned sum;
 
+#ifdef howmany
+#undef howmany
+#endif
+
 #define	howmany(x, y)	(((x) + ((y) - 1)) / (y))
 
     /*
@@ -130,6 +146,116 @@ chop_and_send(char *delta, off_t ctm_size, long max_msg_size, char *mail_alias)
     fclose(dfp);
     }
 
+/* 
+ * Carve our CTM delta into pieces, encode them, and drop them in the
+ * queue dir.
+ *
+ * Basic algorythm:
+ *
+ * - for (each piece)
+ * -   gen. temp. file name (one which the de-queuer will ignore)
+ * -   record in array
+ * -   open temp. file
+ * -   encode delta (including headers) into the temp file
+ * -   close temp. file
+ * - end
+ * - lock queue directory
+ * - foreach (temp. file)
+ * -   rename to the proper filename
+ * - end
+ * - unlock queue directory
+ *
+ * This is probably overkill, but it means that incomplete deltas
+ * don't get mailed, and also reduces the window for lock races
+ * between ctm_smail and the de-queueing process.
+ */
+
+void
+chop_and_queue(char *delta, off_t ctm_size, long max_msg_size, char *queue_dir, char *mail_alias)
+{
+    int npieces, pce, len;
+    long msg_size, exp_size;
+    FILE *sfp, *dfp;
+    unsigned sum;
+    char **tempnames, *tempnam, *sn;
+
+#define	howmany(x, y)	(((x) + ((y) - 1)) / (y))
+
+    /*
+     * Work out how many pieces we need, bearing in mind that each piece
+     * grows by 4/3 when encoded.  We count the newlines too, but ignore
+     * all mail headers and piece headers.  They are a "small" (almost
+     * constant) per message overhead that we make the user worry about. :-)
+     */
+    exp_size = ctm_size * 4 / 3;
+    exp_size += howmany(exp_size, LINE_LENGTH);
+    npieces = howmany(exp_size, max_msg_size);
+    msg_size = howmany(ctm_size, npieces);
+
+#undef howmany
+
+    /*
+     * allocate space for the array of filenames. Try to be portable
+     * by not assuming anything to do with sizeof(char *)
+     */
+    tempnames = malloc(npieces * sizeof(char *));
+    if (tempnames == NULL)
+    {
+	err("malloc for tempnames failed");
+	exit(1);
+    }
+ 
+    len = strlen(queue_dir) + 16;
+    tempnam = malloc(len);
+    if (tempnam == NULL)
+    {
+	err("malloc for tempnames failed");
+	exit(1);
+    }
+    
+    if ((dfp = fopen(delta, "r")) == NULL)
+    {
+	err("cannot open '%s' for reading.", delta);
+	exit(1);
+    }
+
+    if ((sn = strrchr(delta, '/')) == NULL)
+	sn = delta;
+    else
+	sn++;
+
+    for (pce = 1; pce <= npieces; pce++)
+    {
+	if (snprintf(tempnam, len, "%s/.%08d-%03d", queue_dir, getpid(), pce) >= len)
+	    err("Whoops! tempnam isn't long enough");
+
+	tempnames[pce - 1] = strdup(tempnam);
+	if (tempnames[pce - 1] == NULL)
+	{
+	    err("strdup failed for temp. filename");
+	    exit(1);
+	}
+
+	sfp = fopen(tempnam, "w");
+	if (sfp == NULL)
+	    exit(1);
+	
+	write_header(sfp, mail_alias, delta, pce, npieces);
+	sum = encode_body(sfp, dfp, msg_size);
+	write_trailer(sfp, sum);
+
+	if (fclose(sfp) != 0)
+	    exit(1);
+
+	err("%s %d/%d created succesfully", sn, pce, npieces);
+    }
+
+    add_to_queue(queue_dir, mail_alias, delta, npieces, tempnames);
+
+    fclose(dfp);
+
+}
+
 
 /*
  * MIME BASE64 encode table.
@@ -216,7 +342,7 @@ encode_body(FILE *sm_fp, FILE *delta_fp, long msg_size)
 
     if (ferror(sm_fp))
 	{
-	err("error writing to sendmail");
+	err("error writing encoded file");
 	exit(1);
 	}
 
@@ -324,3 +450,138 @@ close_sendmail(FILE *fp)
 
     return (status == 0);
     }
+
+/*
+ * Lock the queuedir so we're the only guy messing about in there.
+ */
+int
+lock_queuedir(char *queue_dir)
+{
+    int fp, len;
+    char *buffer;
+    struct stat sb;
+
+    len = strlen(queue_dir) + 8;
+
+    buffer = malloc(len);
+    if (buffer == NULL)
+    {
+	err("malloc failed in lock_queuedir");
+	exit(1);
+    }
+
+    if (snprintf(buffer, len, "%s/.lock", queue_dir) >= len)
+	err("Whoops. lock buffer too small in lock_queuedir");
+
+    /*
+     * We do our own lockfile scanning to avoid unlink races. 60
+     * seconds should be enough to ensure that we won't get more races
+     * happening between the stat and the open/flock.
+     */
+
+    while (stat(buffer, &sb) == 0)
+	sleep(60);
+
+    if ((fp = open(buffer, O_WRONLY | O_CREAT | O_EXLOCK, 0600)) < 0)
+    {
+	err("can't open `%s' in lock_queuedir", buffer);
+	exit(1);
+    }
+
+    snprintf(buffer, len, "%8ld", getpid());
+    write(fp, buffer, 8);
+
+    free(buffer);
+    
+    return(fp);
+}
+
+/*
+ * Lock the queuedir so we're the only guy messing about in there.
+ */
+void
+free_lock(int lockf, char *queue_dir)
+{
+    int len;
+    char *path;
+
+    /*
+     * Most important: free the lock before we do anything else!
+     */
+
+    close(lockf);
+
+    len = strlen(queue_dir) + 7;
+
+    path = malloc(len);
+    if (path == NULL)
+    {
+	err("malloc failed in free_lock");
+	exit(1);
+    }
+
+    if (snprintf(path, len, "%s/.lock", queue_dir) >= len)
+	err("lock path buffer too small in free_lock");
+
+    if (unlink(path) != 0)
+    {
+	err("can't unlink lockfile `%s'", path);
+	exit(1);
+    }
+
+    free(path);
+}
+
+/* move everything into the queue directory. */
+
+void
+add_to_queue(char *queue_dir, char *mail_alias, char *delta, int npieces, char **tempnames)
+{
+    char *queuefile, *sn;
+    int pce, len, lockf;
+    
+    if ((sn = strrchr(delta, '/')) == NULL)
+	sn = delta;
+    else
+	sn++;
+
+    /* try to malloc all we need BEFORE entering the lock loop */
+    
+    len = strlen(queue_dir) + strlen(sn) + 7;
+    queuefile = malloc(len);
+    if (queuefile == NULL)
+    {
+	err("can't malloc for queuefile");
+	exit(1);
+    }
+
+    /*
+     * We should be the only process mucking around in the queue
+     * directory while we add the new queue files ... it could be
+     * awkward if the de-queue process starts it's job while we're
+     * adding files ...
+     */
+
+    lockf = lock_queuedir(queue_dir);
+    for (pce = 0; pce < npieces; pce++)
+    {
+	struct stat sb;
+
+	if (snprintf(queuefile, len, "%s/%s+%03d", queue_dir, sn, pce + 1) >= len)
+	    err("whoops, queuefile buffer is too small");
+
+	if (stat(queuefile, &sb) == 0)
+	{
+	    err("WOAH! Queue file `%s' already exists! Bailing out.", queuefile);
+	    free_lock(lockf, queue_dir);
+	    exit(1);
+	}
+
+	rename(tempnames[pce], queuefile);
+	err("Queue file %s now exists", queuefile);
+    }
+    
+    free_lock(lockf, queue_dir);
+
+    free(queuefile);
+}