Update to current reality.

mkctm.c can replace the guts of mkCTM if anybody feels like it...

5 files changed, 482 insertions, 7 deletions

diff --git a/usr.sbin/ctm/mkCTM/Makefile b/usr.sbin/ctm/mkCTM/Makefile
new file mode 100644
index 0000000..39dbad5
--- /dev/null
+++ b/usr.sbin/ctm/mkCTM/Makefile
@@ -0,0 +1,18 @@
+
+PROG=	mkctm
+SRCS=	mkctm.c bdiff.c
+LDADD= -lmd
+CFLAGS= -g -Wall
+
+test:	mkctm
+	rm -f tst.out*
+	time ./mkctm /usr/src /a/stable/src 2>a | md5 -p > ${.CURDIR}/tst.out 
+	ls -l ${.CURDIR}/tst.out
+	gzip -9 -v tst.out
+	ls -l ${.CURDIR}/tst.out.gz
+	# cd /usr/src/release && ctm -c -v -v  ${.CURDIR}/tst.out
+
+test1:	mkctm
+	time ./mkctm 2>&1 | more
+
+.include <bsd.prog.mk>
diff --git a/usr.sbin/ctm/mkCTM/ctm_conf.smp-cur b/usr.sbin/ctm/mkCTM/ctm_conf.smp-cur
new file mode 100644
index 0000000..8d8d59c
--- /dev/null
+++ b/usr.sbin/ctm/mkCTM/ctm_conf.smp-cur
@@ -0,0 +1,7 @@
+#!/usr/local/bin/tclsh
+
+set CTMname	smp-cur
+set CTMref	/home/smp
+set CTMdest	$CTMSW/../CTM-priv/$CTMname
+set CTMdont	{\.core$|^/CVSROOT/history.*$|/#cvs|/\.#}
+set CTMmail	smp-cvs-cur
diff --git a/usr.sbin/ctm/mkCTM/ctm_conf.src-cur b/usr.sbin/ctm/mkCTM/ctm_conf.src-cur
index 9e2fdcb..c24262f 100644
--- a/usr.sbin/ctm/mkCTM/ctm_conf.src-cur
+++ b/usr.sbin/ctm/mkCTM/ctm_conf.src-cur
@@ -1,6 +1,6 @@
 #!/usr/local/bin/tclsh
 
 set CTMname	src-cur
-set CTMref	/usr/src
+set CTMref	/c/src
 set CTMdont	{\.core$|/CVS$|/CVS/|^/secure|^/eBones|/#cvs|/\.#}
 set CTMmail	ctm-src-cur@freebsd.org
diff --git a/usr.sbin/ctm/mkCTM/mkCTM b/usr.sbin/ctm/mkCTM/mkCTM
index 364c977..3f884c1 100644
--- a/usr.sbin/ctm/mkCTM/mkCTM
+++ b/usr.sbin/ctm/mkCTM/mkCTM
@@ -1,4 +1,4 @@
-#!/usr/local/bin/tcl
+#!/usr/local/bin/tclsh7.4
 
 #############################################################################
 ### Add something
@@ -138,6 +138,7 @@ set max_damage 1200
 set damage 0
 set changes 0
 
+exec sh -c "date -u '+%y%m%d%H%M%S $argv'" >> /home/ctm/log
 source $argv
 
 if {$CTMtmp == ""} {
@@ -152,7 +153,7 @@ if {$CTMdest == ""} {
 
 # Make sure we only run one at a time...
 
-set CTMlock Lck.${CTMname}.${CTMdate}.[id process]
+set CTMlock Lck.${CTMname}.${CTMdate}.[pid]
 exec rm -f ${CTMlock}
 exec echo starting > ${CTMlock}
 if {[catch "exec ln $CTMlock LCK.$CTMname" a]} {
@@ -277,8 +278,8 @@ while 1 {
 
     if {$damage > $max_damage} {
 	puts "Too much damage: $damage deletes"
-	exec sh -c "rm -f ${CTMtmp}.*"
-        exec rm -f $CTMlock
+		exec sh -c "rm -f ${CTMtmp}.*"
+		exec rm -f $CTMlock
 	exit 0
     }
 
@@ -303,7 +304,7 @@ while 1 {
     puts $fdout "CTM_END " nonewline
     close $fdout ; unset fdout
 
-    exec sh -x -c "rm -f ${CTMtmp}.*" >&@ stdout
+    exec sh -c "rm -f ${CTMtmp}.*" >&@ stdout
 
     if {$CTMtest} {
 	puts "testing, stopping now."
@@ -314,7 +315,7 @@ while 1 {
 	    puts "Applying delta"
 	    flush stdout
 	    exec echo now applying > $CTMlock
-	    exec sh -e -x -c "cd $CTMcopy ; $CTMSW/ctm -v -v -v ${CTMtmp}:${nm}.gz" >&@ stdout
+	    exec sh -e -c "cd $CTMcopy ; $CTMSW/ctm -v -v -v ${CTMtmp}:${nm}.gz" >&@ stdout
 	    exec echo did apply > $CTMlock
     }
     puts "Moving delta"
diff --git a/usr.sbin/ctm/mkCTM/mkctm.c b/usr.sbin/ctm/mkCTM/mkctm.c
new file mode 100644
index 0000000..021349f
--- /dev/null
+++ b/usr.sbin/ctm/mkCTM/mkctm.c
@@ -0,0 +1,449 @@
+/* Still missing:
+ *
+ * Damage counter
+ * Change counter
+ * Time stamp
+ * prefix
+ * cmd-line args
+ * %100 deltas
+ * delta and Add are different. delta -> Equ.
+ *
+ * mkctm 
+ *	-B regex	Bogus
+ *	-I regex	Ignore
+ *	-D int		Damage
+ *	-v		increase verbosity
+ *	-l str		control logging.
+ *	name		cvs-cur
+ *	prefix		src/secure
+ *	dir1		"Soll"
+ *	dir2		"Ist"
+ *
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <dirent.h>
+#include <regex.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <md5.h>
+#include <err.h>
+#include <signal.h>
+
+#define DEFAULT_IGNORE	"/CVS$|/\\.#"
+#define DEFAULT_BOGUS	"\\.core$"
+regex_t reg_ignore,  reg_bogus;
+int	flag_ignore, flag_bogus;
+
+u_long s1_ignored,	s2_ignored;
+u_long s1_bogus,	s2_bogus;
+u_long s1_wrong,	s2_wrong;
+u_long s_same_dirs,	s_same_files,	s_same_bytes;
+u_long 			s_files_chg,	s_bytes_add,	s_bytes_del;
+u_long s_new_dirs,	s_new_files,	s_new_bytes;
+u_long s_del_dirs,	s_del_files,	s_del_bytes;
+u_long            	s_chg_files,	s_chg_bytes;
+u_long 			s_edit_files,	s_edit_bytes,	s_edit_saves;
+u_long 			s_sub_files,	s_sub_bytes;
+
+void
+print_stat(FILE *fd, char *pre)
+{
+    fprintf(fd,"%sAvoided:\n",pre);
+    fprintf(fd,"%s  ignore:  %5lu old   %5lu new\n",
+	    pre, s1_ignored, s2_ignored);
+    fprintf(fd,"%s  bogus:   %5lu old   %5lu new\n",
+	    pre, s1_bogus, s2_bogus);
+    fprintf(fd,"%s  wrong:   %5lu old   %5lu new\n", 
+	    pre, s1_wrong, s2_wrong);
+    fprintf(fd,"%sDelta:\n",pre);
+    fprintf(fd,"%s  new:     %5lu dirs  %5lu files  %9lu plus\n", 
+	    pre, s_new_dirs, s_new_files, s_new_bytes);
+    fprintf(fd,"%s  del:     %5lu dirs  %5lu files                   %9lu minus\n", 
+	    pre, s_del_dirs, s_del_files, s_del_bytes);
+    fprintf(fd,"%s  chg:                 %5lu files  %9lu plus   %9lu minus\n",
+	    pre, s_files_chg, s_bytes_add, s_bytes_del);
+    fprintf(fd,"%s  same:    %5lu dirs  %5lu files  %9lu bytes\n", 
+	    pre, s_same_dirs, s_same_files, s_same_bytes);
+    fprintf(fd,"%sMethod:\n",pre);
+    fprintf(fd,"%s  edit:                %5lu files  %9lu bytes  %9lu saved\n", 
+	    pre, s_edit_files, s_edit_bytes, s_edit_saves);
+    fprintf(fd,"%s  sub:                 %5lu files  %9lu bytes\n", 
+	    pre, s_sub_files, s_sub_bytes);
+}
+
+void
+stat_info(int foo)
+{
+	signal(SIGINFO,stat_info);
+	print_stat(stderr,"INFO: ");
+}
+
+void DoDir(const char *dir1, const char *dir2, const char *name);
+
+static struct stat st;
+static __inline struct stat *
+StatFile(char *name)
+{
+	if (lstat(name,&st) < 0) 
+		err(1,"Couldn't stat %s\n",name);
+	return &st;
+}
+
+int
+dirselect(struct dirent *de)
+{
+	if (!strcmp(de->d_name,"."))	return 0;
+	if (!strcmp(de->d_name,".."))	return 0;
+	return 1;
+}
+
+void
+name_stat(const char *pfx, const char *dir, const char *name, struct dirent *de)
+{
+	char *buf = alloca(strlen(dir) + strlen(name) + 
+		strlen(de->d_name) + 3);
+	struct stat *st;
+
+	strcpy(buf,dir); 
+		strcat(buf,"/"); strcat(buf,name);
+		strcat(buf,"/"); strcat(buf,de->d_name);
+	st = StatFile(buf);
+	printf("%s %s%s %lu %lu %o", 
+	    pfx, name, de->d_name, 
+	    st->st_uid, st->st_gid, st->st_mode & ~S_IFMT);
+}
+
+void
+Equ(const char *dir1, const char *dir2, const char *name, struct dirent *de)
+{
+	if (de->d_type == DT_DIR) {
+		char *p = alloca(strlen(name)+strlen(de->d_name)+2);
+
+		strcpy(p,name);  strcat(p,de->d_name); strcat(p, "/");
+		DoDir(dir1,dir2,p);
+		s_same_dirs++;
+	} else {
+		char *buf1 = alloca(strlen(dir1) + strlen(name) + 
+			strlen(de->d_name) + 3);
+		char *buf2 = alloca(strlen(dir2) + strlen(name) + 
+			strlen(de->d_name) + 3);
+		char *m1,md5_1[33],*m2, md5_2[33];
+		void *p1,*p2;
+		int fd1,fd2;
+		struct stat s1,s2;
+
+		strcpy(buf1,dir1); 
+			strcat(buf1,"/"); strcat(buf1,name);
+			strcat(buf1,"/"); strcat(buf1,de->d_name);
+		fd1 = open(buf1,O_RDONLY);
+		if(fd1 < 0) { perror(buf1); exit(3); }
+		fstat(fd1,&s1);
+		strcpy(buf2,dir2); 
+			strcat(buf2,"/"); strcat(buf2,name);
+			strcat(buf2,"/"); strcat(buf2,de->d_name);
+		fd2 = open(buf2,O_RDONLY);
+		if(fd2 < 0) { perror(buf2); exit(3); }
+		fstat(fd2,&s2);
+#if 1
+		if (s1.st_size == s2.st_size) {
+			s_same_files++;
+			s_same_bytes += s1.st_size;
+			close(fd1);
+			close(fd2);
+			goto finish;
+		}
+#endif
+		p1=mmap(0,s1.st_size,PROT_READ,MAP_PRIVATE,fd1,0);
+		if ((int)p1 == -1) { perror(buf1); exit(3); }
+		close(fd1);
+
+		p2=mmap(0,s2.st_size,PROT_READ,MAP_PRIVATE,fd2,0);
+		if ((int)p2 == -1) { perror(buf2); exit(3); }
+		close(fd2);
+
+		/* If identical, we're done. */
+		if((s1.st_size == s2.st_size) && !memcmp(p1,p2,s1.st_size)) {
+			s_same_files++;
+			s_same_bytes += s1.st_size;
+			goto finish;
+		}
+
+		s_files_chg++;
+		if (s1.st_size > s2.st_size)
+			s_bytes_del += (s1.st_size - s2.st_size);
+		else
+			s_bytes_add += (s2.st_size - s1.st_size);
+
+		m1 = MD5Data(p1, s1.st_size, md5_1);
+		m2 = MD5Data(p2, s2.st_size, md5_2);
+		
+		/* Just a curiosity... */
+		if(!strcmp(m1,m2)) {
+			if (s1.st_size != s2.st_size) 
+				fprintf(stderr,
+		"Notice: MD5 same for files of diffent size:\n\t%s\n\t%s\n",
+					buf1,buf2);
+			goto finish;
+		}
+
+		{
+			u_long l = s1.st_size + s2.st_size;
+			u_char *ob = alloca(l);
+			Dissect(p1,p1+s1.st_size,p2,p2+s2.st_size,ob,&l);
+			if (l && l < s2.st_size) {
+				name_stat("CTMFB",dir2,name,de);
+				printf(" %s %s %d\n",m1,m2,(unsigned)s1.st_size);
+				s_edit_files++;
+				s_edit_bytes += l;
+				s_edit_saves += (s2.st_size - l);
+				fwrite(ob,1,l,stdout);
+				putchar('\n');
+			} else {
+				name_stat("CTMFS",dir2,name,de);
+				printf(" %s %s %u\n",m1,m2,(unsigned)s2.st_size);
+				fwrite(p2,1,s2.st_size,stdout);
+				s_sub_files++;
+				s_sub_bytes += s2.st_size;
+			}
+		}
+	    finish:
+		munmap(p1,s1.st_size);
+		munmap(p2,s2.st_size);
+	}
+}
+
+void
+Add(const char *dir1, const char *dir2, const char *name, struct dirent *de)
+{
+	if (de->d_type == DT_DIR) {
+		char *p = alloca(strlen(name)+strlen(de->d_name)+2);
+		strcpy(p,name);  strcat(p,de->d_name); strcat(p, "/");
+		name_stat("CTMDM",dir2,name,de);
+		putchar('\n');
+		s_new_dirs++;
+		DoDir(dir1,dir2,p);
+	} else if (de->d_type == DT_REG) {
+		char *buf2 = alloca(strlen(dir2) + strlen(name) + 
+			strlen(de->d_name) + 3);
+		char *m2, md5_2[33];
+		u_char *p1;
+		struct stat st;
+		int fd1;
+
+		strcpy(buf2,dir2); 
+			strcat(buf2,"/"); strcat(buf2,name);
+			strcat(buf2,"/"); strcat(buf2,de->d_name);
+		fd1 = open(buf2,O_RDONLY);
+		if (fd1 < 0) {perror(buf2); exit (3); }
+		fstat(fd1,&st);
+		p1=mmap(0,st.st_size,PROT_READ,MAP_PRIVATE,fd1,0);
+		if ((int)p1 == -1) { perror(buf2); exit(3); }
+		close(fd1);
+		m2 = MD5Data(p1, st.st_size, md5_2);
+		name_stat("CTMFM",dir2,name,de);
+		printf(" %s %u\n",m2,(unsigned)st.st_size);
+		fwrite(p1,1,st.st_size,stdout);
+		putchar('\n');
+		munmap(p1,st.st_size);
+		s_new_files++;
+		s_new_bytes += st.st_size;
+	}
+}
+
+void
+Del (const char *dir1, const char *dir2, const char *name, struct dirent *de)
+{
+	if (de->d_type == DT_DIR) {
+		char *p = alloca(strlen(name)+strlen(de->d_name)+2);
+		strcpy(p,name);  strcat(p,de->d_name); strcat(p, "/");
+		DoDir(dir1,dir2,p);
+		printf("CTMDR %s%s\n",name,de->d_name);
+		s_del_dirs++;
+	} else if (de->d_type == DT_REG) {
+		char *buf1 = alloca(strlen(dir1) + strlen(name) + 
+			strlen(de->d_name) + 3);
+		char *m1, md5_1[33];
+		strcpy(buf1,dir1); 
+			strcat(buf1,"/"); strcat(buf1,name);
+			strcat(buf1,"/"); strcat(buf1,de->d_name);
+		m1 = MD5File(buf1, md5_1);
+		printf("CTMFR %s%s %s\n",name,de->d_name,m1);
+		s_del_files++;
+		s_del_bytes += StatFile(buf1)->st_size;
+	}
+}
+
+void
+GetNext(int *i, int *n, struct dirent **nl, const char *dir, const char *name, u_long *ignored, u_long *bogus, u_long *wrong)
+{
+	char buf[BUFSIZ];
+
+	for (;;) {
+		for (;;) {
+			(*i)++;
+			if (*i >= *n)
+				return;
+			*buf = 0;
+			if (*dir != '/')
+				strcat(buf,"/");
+			strcat(buf,dir); 
+			if (buf[strlen(buf)-1] != '/')
+				strcat(buf,"/"); 
+			strcat(buf,name);
+			if (buf[strlen(buf)-1] != '/')
+				strcat(buf,"/"); 
+			fprintf(stderr,">%d<%s>\n",strlen(nl[*i]->d_name),nl[*i]->d_name);
+			fflush(stderr);
+			strcat(buf,nl[*i]->d_name);
+			if (flag_ignore && !regexec(&reg_ignore,buf,0,0,0))
+				(*ignored)++;
+			else if (flag_bogus && !regexec(&reg_bogus,buf,0,0,0))
+				(*bogus)++;
+			else
+				break;
+			free(nl[*i]); nl[*i] = 0;
+		}
+		/* If the filesystem didn't tell us, find type */
+		if (nl[*i]->d_type == DT_UNKNOWN) 
+			nl[*i]->d_type = IFTODT(StatFile(buf)->st_mode);
+		if (nl[*i]->d_type == DT_REG || nl[*i]->d_type == DT_DIR)
+			break;
+		(*wrong)++;
+		free(nl[*i]); nl[*i] = 0;
+	}
+}
+
+void
+DoDir(const char *dir1, const char *dir2, const char *name)
+{
+	int i1,i2,n1,n2,i;
+	struct dirent **nl1,**nl2;
+	char *buf1 = alloca(strlen(dir1) + strlen(name) + 4);
+	char *buf2 = alloca(strlen(dir2) + strlen(name) + 4);
+
+	strcpy(buf1,dir1); strcat(buf1,"/"); strcat(buf1,name);
+	strcpy(buf2,dir2); strcat(buf2,"/"); strcat(buf2,name);
+	n1 = scandir(buf1, &nl1, dirselect, alphasort);
+	n2 = scandir(buf2, &nl2, dirselect, alphasort);
+	i1 = i2 = -1;
+	GetNext(&i1, &n1, nl1, dir1, name, &s1_ignored, &s1_bogus, &s1_wrong);
+	GetNext(&i2, &n2, nl2, dir2, name, &s2_ignored, &s2_bogus, &s2_wrong);
+	for (;i1 < n1 || i2 < n2;) {
+
+		/* Get next item from list 1 */
+		if (i1 < n1 && !nl1[i1]) 
+			GetNext(&i1, &n1, nl1, dir1, name, 
+				&s1_ignored, &s1_bogus, &s1_wrong);
+
+		/* Get next item from list 2 */
+		if (i2 < n2 && !nl2[i2]) 
+			GetNext(&i2, &n2, nl2, dir2, name, 
+				&s2_ignored, &s2_bogus, &s2_wrong);
+
+		if (i1 >= n1 && i2 >= n2) {
+			/* Done */
+			break;
+		} else if (i1 >= n1 && i2 < n2) {
+			/* end of list 1, add anything left on list 2 */
+			Add(dir1,dir2,name,nl2[i2]);
+			free(nl2[i2]); nl2[i2] = 0;
+		} else if (i1 < n1 && i2 >= n2) {
+			/* end of list 2, delete anything left on list 1 */
+			Del(dir1,dir2,name,nl1[i1]);
+			free(nl1[i1]); nl1[i1] = 0;
+		} else if (!(i = strcmp(nl1[i1]->d_name, nl2[i2]->d_name))) {
+			/* Identical names */
+			if (nl1[i1]->d_type == nl2[i2]->d_type) {
+				/* same type */
+				Equ(dir1,dir2,name,nl1[i1]);
+			} else {
+				/* different types */
+				Del(dir1,dir2,name,nl1[i1]);
+				Add(dir1,dir2,name,nl2[i2]);
+			}
+			free(nl1[i1]); nl1[i1] = 0;
+			free(nl2[i2]); nl2[i2] = 0;
+		} else if (i < 0) {
+			/* Something extra in list 1, delete it */
+			Del(dir1,dir2,name,nl1[i1]);
+			free(nl1[i1]); nl1[i1] = 0;
+		} else {
+			/* Something extra in list 2, add it */
+			Add(dir1,dir2,name,nl2[i2]);
+			free(nl2[i2]); nl2[i2] = 0;
+		}
+	}
+	if (n1 >= 0)
+		free(nl1);
+	if (n2 >= 0)
+		free(nl2);
+}
+
+int
+main(int argc, char **argv)
+{
+	int i;
+	extern char *optarg;
+	extern int optind;
+
+	if (regcomp(&reg_bogus,DEFAULT_BOGUS, REG_EXTENDED | REG_NEWLINE))
+		/* XXX use regerror to explain it */
+		err(1,"Default regular expression argument to -B is botched");
+	flag_bogus = 1;
+
+	if (regcomp(&reg_ignore,DEFAULT_IGNORE, REG_EXTENDED | REG_NEWLINE))
+		/* XXX use regerror to explain it */
+		err(1,"Default regular expression argument to -I is botched");
+	flag_ignore = 1;
+
+	while ((i = getopt(argc,argv,"I:B:")) != EOF)
+		switch (i) {
+		case 'I':
+			if (flag_ignore)
+				regfree(&reg_ignore);
+			flag_ignore = 0;
+			if (!*optarg)
+				break;
+			if (regcomp(&reg_ignore,optarg,
+			    REG_EXTENDED | REG_NEWLINE))
+				/* XXX use regerror to explain it */
+				err(1,"Regular expression argument to -I is botched");
+			flag_ignore = 1;
+			break;
+		case 'B':
+			if (flag_bogus)
+				regfree(&reg_bogus);
+			flag_bogus = 0;
+			if (!*optarg)
+				break;
+			if (regcomp(&reg_bogus,optarg,
+			    REG_EXTENDED | REG_NEWLINE))
+				/* XXX use regerror to explain it */
+				err(1,"Regular expression argument to -B is botched");
+			flag_bogus = 1;
+			break;
+		case '?':
+		default:
+			fprintf(stderr,"Usage:\n\t%s: %s\n", argv[0],
+"[-I ignore_re] [-B bogus_re]");
+			return (1);
+		}
+	argc -= optind;
+	argv += optind;
+
+	setbuf(stdout,0);
+
+	signal(SIGINFO,stat_info);
+	printf("CTM_BEGIN 2.0 tst 0 950326022230Z .\n");
+	DoDir(argv[0],argv[1],"");
+	printf("CTM_END ");
+	print_stat(stderr,"");
+	exit(0);
+}