Reintegrate colldef(1) and mklocale(1)

While those tools are not needed anymore they are necessary to build FreeBSD 9
and 10. it does not hurt to keep those tools around until both 9 and 10 branch
become EOLed.

Modify colldef(1) to build after the change in the collation header, and ensure
it does produce the same collation definition it used to generate for 9 and 10

Reported by:	Oliver Pinter

6 files changed, 1011 insertions, 0 deletions

diff --git a/usr.bin/colldef/Makefile b/usr.bin/colldef/Makefile
new file mode 100644
index 0000000..fd59a6c
--- /dev/null
+++ b/usr.bin/colldef/Makefile
@@ -0,0 +1,12 @@
+# $FreeBSD$
+
+PROG=	colldef
+SRCS=	parse.y scan.l y.tab.h
+LFLAGS=	-8 -i
+CFLAGS+=-I. -I${.CURDIR} -I${.CURDIR}/../../lib/libc/locale
+CFLAGS+=-DCOLLATE_DEBUG -DYY_NO_UNPUT -DYY_NO_INPUT
+LIBADD=	l
+
+WARNS?=	2
+
+.include <bsd.prog.mk>
diff --git a/usr.bin/colldef/Makefile.depend b/usr.bin/colldef/Makefile.depend
new file mode 100644
index 0000000..3ca32c6
--- /dev/null
+++ b/usr.bin/colldef/Makefile.depend
@@ -0,0 +1,27 @@
+# $FreeBSD$
+# Autogenerated - do NOT edit!
+
+DIRDEPS = \
+	gnu/lib/csu \
+	gnu/lib/libgcc \
+	include \
+	include/arpa \
+	include/xlocale \
+	lib/${CSU_DIR} \
+	lib/libc \
+	lib/libcompiler_rt \
+	usr.bin/lex/lib \
+	usr.bin/yacc.host \
+
+
+.include <dirdeps.mk>
+
+.if ${DEP_RELDIR} == ${_DEP_RELDIR}
+# local dependencies - needed for -jN in clean tree
+parse.o: parse.c
+parse.po: parse.c
+scan.o: scan.c
+scan.o: y.tab.h
+scan.po: scan.c
+scan.po: y.tab.h
+.endif
diff --git a/usr.bin/colldef/colldef.1 b/usr.bin/colldef/colldef.1
new file mode 100644
index 0000000..b541cd9
--- /dev/null
+++ b/usr.bin/colldef/colldef.1
@@ -0,0 +1,272 @@
+.\" Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua>
+.\"		at Electronni Visti IA, Kiev, Ukraine.
+.\"			All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd January 27, 1995
+.Dt COLLDEF 1
+.Os
+.Sh NAME
+.Nm colldef
+.Nd convert collation sequence source definition
+.Sh SYNOPSIS
+.Nm
+.Op Fl I Ar map_dir
+.Op Fl o Ar out_file
+.Op Ar filename
+.Sh DESCRIPTION
+The
+.Nm
+utility converts a collation sequence source definition
+into a format usable by the
+.Fn strxfrm
+and
+.Fn strcoll
+functions.
+It is used to define the many ways in which
+strings can be ordered and collated.
+The
+.Fn strxfrm
+function transforms
+its first argument and places the result in its second
+argument.
+The transformed string is such that it can be
+correctly ordered with other transformed strings by using
+.Fn strcmp ,
+.Fn strncmp ,
+or
+.Fn memcmp .
+The
+.Fn strcoll
+function transforms its arguments and does a
+comparison.
+.Pp
+The
+.Nm
+utility reads the collation sequence source definition
+from the standard input and stores the converted definition in filename.
+The output file produced contains the
+database with collating sequence information in a form
+usable by system commands and routines.
+.Pp
+The following options are available:
+.Bl -tag -width indent
+.It Fl I Ar map_dir
+Set directory name where
+.Ar charmap
+files can be found, current directory by default.
+.It Fl o Ar out_file
+Set output file name,
+.Ar LC_COLLATE
+by default.
+.El
+.Pp
+The collation sequence definition specifies a set of collating elements and
+the rules defining how strings containing these should be ordered.
+This is most useful for different language definitions.
+.Pp
+The specification file can consist of three statements:
+.Ar charmap ,
+.Ar substitute
+and
+.Ar order .
+.Pp
+Of these, only the
+.Ar order
+statement is required.
+When
+.Ar charmap
+or
+.Ar substitute
+is
+supplied, these statements must be ordered as above.
+Any
+statements after the order statement are ignored.
+.Pp
+Lines in the specification file beginning with a
+.Ql #
+are
+treated as comments and are ignored.
+Blank lines are also
+ignored.
+.Pp
+.Dl "charmap charmapfile"
+.Pp
+.Ar Charmap
+defines where a mapping of the character
+and collating element symbols to the actual
+character encoding can be found.
+.Pp
+The format of
+.Ar charmapfile
+is shown below.
+Symbol
+names are separated from their values by TAB or
+SPACE characters.
+Symbol-value can be specified in
+a hexadecimal (\ex\fI??\fR) or octal (\e\fI???\fR)
+representation, and can be only one character in length.
+.Bd -literal -offset indent
+symbol-name1 symbol-value1
+symbol-name2 symbol-value2
+\&...
+.Ed
+.Pp
+Symbol names cannot be specified in
+.Ar substitute
+fields.
+.Pp
+The
+.Ar charmap
+statement is optional.
+.Bd -literal -offset indent
+substitute "symbol" with "repl_string"
+.Ed
+.Pp
+The
+.Ar substitute
+statement substitutes the character
+.Ar symbol
+with the string
+.Ar repl_string .
+Symbol names cannot be specified in
+.Ar repl_string
+field.
+The
+.Ar substitute
+statement is optional.
+.Pp
+.Dl "order order_list"
+.Pp
+.Ar Order_list
+is a list of symbols, separated by semi colons, that defines the
+collating sequence.
+The
+special symbol
+.Ar ...
+specifies, in a short-hand
+form, symbols that are sequential in machine code
+order.
+.Pp
+An order list element
+can be represented in any one of the following
+ways:
+.Bl -bullet
+.It
+The symbol itself (for example,
+.Ar a
+for the lower-case letter
+.Ar a ) .
+.It
+The symbol in octal representation (for example,
+.Ar \e141
+for the letter
+.Ar a ) .
+.It
+The symbol in hexadecimal representation (for example,
+.Ar \ex61
+for the letter
+.Ar a ) .
+.It
+The symbol name as defined in the
+.Ar charmap
+file (for example,
+.Ar <letterA>
+for
+.Ar letterA \e023
+record in
+.Ar charmapfile ) .
+If character map name have
+.Ar >
+character, it must be escaped as
+.Ar /> ,
+single
+.Ar /
+must be escaped as
+.Ar // .
+.It
+Symbols
+.Ar \ea ,
+.Ar \eb ,
+.Ar \ef ,
+.Ar \en ,
+.Ar \er ,
+.Ar \ev
+are permitted in its usual C-language meaning.
+.It
+The symbol chain (for example:
+.Ar abc ,
+.Ar <letterA><letterB>c ,
+.Ar \exf1b\exf2 )
+.It
+The symbol range (for example,
+.Ar a;...;z ) .
+.It
+Comma-separated symbols, ranges and chains enclosed in parenthesis (for example
+.Ar \&(
+.Ar sym1 ,
+.Ar sym2 ,
+.Ar ...
+.Ar \&) )
+are assigned the
+same primary ordering but different secondary
+ordering.
+.It
+Comma-separated symbols, ranges and chains enclosed in curly brackets (for example
+.Ar \&{
+.Ar sym1 ,
+.Ar sym2 ,
+.Ar ...
+.Ar \&} )
+are assigned the same primary ordering only.
+.El
+.Pp
+The backslash character
+.Ar \e
+is used for continuation.
+In this case, no characters are permitted
+after the backslash character.
+.Sh FILES
+.Bl -tag -width indent
+.It Pa /usr/share/locale/ Ns Ao Ar language Ac Ns Pa /LC_COLLATE
+The standard shared location for collation orders
+under the locale
+.Aq Ar language .
+.El
+.Sh EXIT STATUS
+The
+.Nm
+utility exits with the following values:
+.Bl -tag -width indent
+.It Li 0
+No errors were found and the output was successfully created.
+.It Li !=0
+Errors were found.
+.El
+.Sh SEE ALSO
+.Xr mklocale 1 ,
+.Xr setlocale 3 ,
+.Xr strcoll 3 ,
+.Xr strxfrm 3
diff --git a/usr.bin/colldef/common.h b/usr.bin/colldef/common.h
new file mode 100644
index 0000000..316490d
--- /dev/null
+++ b/usr.bin/colldef/common.h
@@ -0,0 +1,11 @@
+/*
+ * $FreeBSD$
+ */
+
+#define CHARMAP_SYMBOL_LEN 64
+#define BUFSIZE 80
+
+extern int line_no;
+
+extern u_char charmap_table[UCHAR_MAX + 1][CHARMAP_SYMBOL_LEN];
+extern char map_name[FILENAME_MAX];
diff --git a/usr.bin/colldef/parse.y b/usr.bin/colldef/parse.y
new file mode 100644
index 0000000..1c04874
--- /dev/null
+++ b/usr.bin/colldef/parse.y
@@ -0,0 +1,402 @@
+%{
+/*-
+ * Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua>
+ *		at Electronni Visti IA, Kiev, Ukraine.
+ *			All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <arpa/inet.h>
+#include <err.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sysexits.h>
+#include "collate.h"
+#include "common.h"
+
+extern FILE *yyin;
+void yyerror(const char *fmt, ...) __printflike(1, 2);
+int yyparse(void);
+int yylex(void);
+static void usage(void);
+static void collate_print_tables(void);
+
+#undef STR_LEN
+#define STR_LEN 10
+#undef TABLE_SIZE
+#define TABLE_SIZE 100
+#undef COLLATE_VERSION
+#define COLLATE_VERSION    "1.0\n"
+#undef COLLATE_VERSION_2
+#define COLLATE_VERSION1_2 "1.2\n"
+
+struct __collate_st_char_pri {
+	int prim, sec;
+};
+
+struct __collate_st_chain_pri {
+	u_char str[STR_LEN];
+	int prim, sec;
+};
+
+char map_name[FILENAME_MAX] = ".";
+char curr_chain[STR_LEN];
+
+char __collate_version[STR_LEN];
+u_char charmap_table[UCHAR_MAX + 1][CHARMAP_SYMBOL_LEN];
+
+#undef __collate_substitute_table
+u_char __collate_substitute_table[UCHAR_MAX + 1][STR_LEN];
+#undef __collate_char_pri_table
+struct __collate_st_char_pri __collate_char_pri_table[UCHAR_MAX + 1];
+struct __collate_st_chain_pri *__collate_chain_pri_table;
+
+int chain_index = 0;
+int prim_pri = 1, sec_pri = 1;
+#ifdef COLLATE_DEBUG
+int debug;
+#endif
+
+const char *out_file = "LC_COLLATE";
+%}
+%union {
+	u_char ch;
+	u_char str[BUFSIZE];
+}
+%token SUBSTITUTE WITH ORDER RANGE
+%token <str> STRING
+%token <str> DEFN
+%token <ch> CHAR
+%%
+collate : statment_list
+;
+statment_list : statment
+	| statment_list '\n' statment
+;
+statment :
+	| charmap
+	| substitute
+	| order
+;
+charmap : DEFN CHAR {
+	if (strlen($1) + 1 > CHARMAP_SYMBOL_LEN)
+		yyerror("Charmap symbol name '%s' is too long", $1);
+	strcpy(charmap_table[$2], $1);
+}
+;
+substitute : SUBSTITUTE CHAR WITH STRING {
+	if ($2 == '\0')
+		yyerror("NUL character can't be substituted");
+	if (strchr($4, $2) != NULL)
+		yyerror("Char 0x%02x substitution is recursive", $2);
+	if (strlen($4) + 1 > STR_LEN)
+		yyerror("Char 0x%02x substitution is too long", $2);
+	strcpy(__collate_substitute_table[$2], $4);
+}
+;
+order : ORDER order_list {
+	FILE *fp;
+	int ch, substed, ordered;
+	uint32_t u32;
+
+	for (ch = 0; ch < UCHAR_MAX + 1; ch++) {
+		substed = (__collate_substitute_table[ch][0] != ch);
+		ordered = !!__collate_char_pri_table[ch].prim;
+		if (!ordered && !substed)
+			yyerror("Char 0x%02x not found", ch);
+		if (substed && ordered)
+			yyerror("Char 0x%02x can't be ordered since substituted", ch);
+	}
+
+	if ((__collate_chain_pri_table = realloc(__collate_chain_pri_table,
+	     sizeof(*__collate_chain_pri_table) * (chain_index + 1))) == NULL)
+		yyerror("can't grow chain table");
+	(void)memset(&__collate_chain_pri_table[chain_index], 0,
+		     sizeof(__collate_chain_pri_table[0]));
+	chain_index++;
+
+#ifdef COLLATE_DEBUG
+	if (debug)
+		collate_print_tables();
+#endif
+	if ((fp = fopen(out_file, "w")) == NULL)
+		err(EX_UNAVAILABLE, "can't open destination file %s",
+		    out_file);
+
+	strcpy(__collate_version, COLLATE_VERSION1_2);
+	if (fwrite(__collate_version, sizeof(__collate_version), 1, fp) != 1)
+		err(EX_IOERR,
+		"I/O error writing collate version to destination file %s",
+		    out_file);
+	u32 = htonl(chain_index);
+	if (fwrite(&u32, sizeof(u32), 1, fp) != 1)
+		err(EX_IOERR,
+		"I/O error writing chains number to destination file %s",
+		    out_file);
+	if (fwrite(__collate_substitute_table,
+		   sizeof(__collate_substitute_table), 1, fp) != 1)
+		err(EX_IOERR,
+		"I/O error writing substitution table to destination file %s",
+		    out_file);
+	for (ch = 0; ch < UCHAR_MAX + 1; ch++) {
+		__collate_char_pri_table[ch].prim =
+		    htonl(__collate_char_pri_table[ch].prim);
+		__collate_char_pri_table[ch].sec =
+		    htonl(__collate_char_pri_table[ch].sec);
+	}
+	if (fwrite(__collate_char_pri_table,
+		   sizeof(__collate_char_pri_table), 1, fp) != 1)
+		err(EX_IOERR,
+		"I/O error writing char table to destination file %s",
+		    out_file);
+	for (ch = 0; ch < chain_index; ch++) {
+		__collate_chain_pri_table[ch].prim =
+		    htonl(__collate_chain_pri_table[ch].prim);
+		__collate_chain_pri_table[ch].sec =
+		    htonl(__collate_chain_pri_table[ch].sec);
+	}
+	if (fwrite(__collate_chain_pri_table,
+		   sizeof(*__collate_chain_pri_table), chain_index, fp) !=
+		   (size_t)chain_index)
+		err(EX_IOERR,
+		"I/O error writing chain table to destination file %s",
+		    out_file);
+	if (fclose(fp) != 0)
+		err(EX_IOERR, "I/O error closing destination file %s",
+		    out_file);
+	exit(EX_OK);
+}
+;
+order_list : item
+	| order_list ';' item
+;
+chain : CHAR CHAR {
+	curr_chain[0] = $1;
+	curr_chain[1] = $2;
+	if (curr_chain[0] == '\0' || curr_chain[1] == '\0')
+		yyerror("\\0 can't be chained");
+	curr_chain[2] = '\0';
+}
+	| chain CHAR {
+	static char tb[2];
+
+	tb[0] = $2;
+	if (tb[0] == '\0')
+		yyerror("\\0 can't be chained");
+	if (strlen(curr_chain) + 2 > STR_LEN)
+		yyerror("Chain '%s' grows too long", curr_chain);
+	(void)strcat(curr_chain, tb);
+}
+;
+item :  CHAR {
+	if (__collate_char_pri_table[$1].prim)
+		yyerror("Char 0x%02x duplicated", $1);
+	__collate_char_pri_table[$1].prim = prim_pri++;
+}
+	| chain {
+	if ((__collate_chain_pri_table = realloc(__collate_chain_pri_table,
+	     sizeof(*__collate_chain_pri_table) * (chain_index + 1))) == NULL)
+		yyerror("can't grow chain table");
+	(void)memset(&__collate_chain_pri_table[chain_index], 0,
+		     sizeof(__collate_chain_pri_table[0]));
+	(void)strcpy(__collate_chain_pri_table[chain_index].str, curr_chain);
+	__collate_chain_pri_table[chain_index].prim = prim_pri++;
+	chain_index++;
+}
+	| CHAR RANGE CHAR {
+	u_int i;
+
+	if ($3 <= $1)
+		yyerror("Illegal range 0x%02x -- 0x%02x", $1, $3);
+
+	for (i = $1; i <= $3; i++) {
+		if (__collate_char_pri_table[(u_char)i].prim)
+			yyerror("Char 0x%02x duplicated", (u_char)i);
+		__collate_char_pri_table[(u_char)i].prim = prim_pri++;
+	}
+}
+	| '{' prim_order_list '}' {
+	prim_pri++;
+}
+	| '(' sec_order_list ')' {
+	prim_pri++;
+	sec_pri = 1;
+}
+;
+prim_order_list : prim_sub_item
+	| prim_order_list ',' prim_sub_item 
+;
+sec_order_list : sec_sub_item
+	| sec_order_list ',' sec_sub_item 
+;
+prim_sub_item : CHAR {
+	if (__collate_char_pri_table[$1].prim)
+		yyerror("Char 0x%02x duplicated", $1);
+	__collate_char_pri_table[$1].prim = prim_pri;
+}
+	| CHAR RANGE CHAR {
+	u_int i;
+
+	if ($3 <= $1)
+		yyerror("Illegal range 0x%02x -- 0x%02x",
+			$1, $3);
+
+	for (i = $1; i <= $3; i++) {
+		if (__collate_char_pri_table[(u_char)i].prim)
+			yyerror("Char 0x%02x duplicated", (u_char)i);
+		__collate_char_pri_table[(u_char)i].prim = prim_pri;
+	}
+}
+	| chain {
+	if ((__collate_chain_pri_table = realloc(__collate_chain_pri_table,
+	     sizeof(*__collate_chain_pri_table) * (chain_index + 1))) == NULL)
+		yyerror("can't grow chain table");
+	(void)memset(&__collate_chain_pri_table[chain_index], 0,
+		     sizeof(__collate_chain_pri_table[0]));
+	(void)strcpy(__collate_chain_pri_table[chain_index].str, curr_chain);
+	__collate_chain_pri_table[chain_index].prim = prim_pri;
+	chain_index++;
+}
+;
+sec_sub_item : CHAR {
+	if (__collate_char_pri_table[$1].prim)
+		yyerror("Char 0x%02x duplicated", $1);
+	__collate_char_pri_table[$1].prim = prim_pri;
+	__collate_char_pri_table[$1].sec = sec_pri++;
+}
+	| CHAR RANGE CHAR {
+	u_int i;
+
+	if ($3 <= $1)
+		yyerror("Illegal range 0x%02x -- 0x%02x",
+			$1, $3);
+
+	for (i = $1; i <= $3; i++) {
+		if (__collate_char_pri_table[(u_char)i].prim)
+			yyerror("Char 0x%02x duplicated", (u_char)i);
+		__collate_char_pri_table[(u_char)i].prim = prim_pri;
+		__collate_char_pri_table[(u_char)i].sec = sec_pri++;
+	}
+}
+	| chain {
+	if ((__collate_chain_pri_table = realloc(__collate_chain_pri_table,
+	     sizeof(*__collate_chain_pri_table) * (chain_index + 1))) == NULL)
+		yyerror("can't grow chain table");
+	(void)memset(&__collate_chain_pri_table[chain_index], 0,
+		     sizeof(__collate_chain_pri_table[0]));
+	(void)strcpy(__collate_chain_pri_table[chain_index].str, curr_chain);
+	__collate_chain_pri_table[chain_index].prim = prim_pri;
+	__collate_chain_pri_table[chain_index].sec = sec_pri++;
+	chain_index++;
+}
+;
+%%
+int
+main(int ac, char **av)
+{
+	int ch;
+
+#ifdef COLLATE_DEBUG
+	while((ch = getopt(ac, av, ":do:I:")) != -1) {
+#else
+	while((ch = getopt(ac, av, ":o:I:")) != -1) {
+#endif
+		switch (ch)
+		{
+#ifdef COLLATE_DEBUG
+		  case 'd':
+			debug++;
+			break;
+#endif
+		  case 'o':
+			out_file = optarg;
+			break;
+
+		  case 'I':
+			strlcpy(map_name, optarg, sizeof(map_name));
+			break;
+
+		  default:
+			usage();
+		}
+	}
+	ac -= optind;
+	av += optind;
+	if (ac > 0) {
+		if ((yyin = fopen(*av, "r")) == NULL)
+			err(EX_UNAVAILABLE, "can't open source file %s", *av);
+	}
+	for (ch = 0; ch <= UCHAR_MAX; ch++)
+		__collate_substitute_table[ch][0] = ch;
+	yyparse();
+	return 0;
+}
+
+static void
+usage(void)
+{
+	fprintf(stderr, "usage: colldef [-I map_dir] [-o out_file] [filename]\n");
+	exit(EX_USAGE);
+}
+
+void
+yyerror(const char *fmt, ...)
+{
+	va_list ap;
+	char msg[128];
+
+	va_start(ap, fmt);
+	vsnprintf(msg, sizeof(msg), fmt, ap);
+	va_end(ap);
+	errx(EX_UNAVAILABLE, "%s near line %d", msg, line_no);
+}
+
+#ifdef COLLATE_DEBUG
+static void
+collate_print_tables(void)
+{
+	int i;
+
+	printf("Substitute table:\n");
+	for (i = 0; i < UCHAR_MAX + 1; i++)
+	    if (i != *__collate_substitute_table[i])
+		printf("\t'%c' --> \"%s\"\n", i,
+		       __collate_substitute_table[i]);
+	printf("Chain priority table:\n");
+	for (i = 0; i < chain_index - 1; i++)
+		printf("\t\"%s\" : %d %d\n",
+		    __collate_chain_pri_table[i].str,
+		    __collate_chain_pri_table[i].prim,
+		    __collate_chain_pri_table[i].sec);
+	printf("Char priority table:\n");
+	for (i = 0; i < UCHAR_MAX + 1; i++)
+		printf("\t'%c' : %d %d\n", i, __collate_char_pri_table[i].prim,
+		       __collate_char_pri_table[i].sec);
+}
+#endif
diff --git a/usr.bin/colldef/scan.l b/usr.bin/colldef/scan.l
new file mode 100644
index 0000000..b396ed0
--- /dev/null
+++ b/usr.bin/colldef/scan.l
@@ -0,0 +1,287 @@
+%x string name charmap defn nchar subs subs2
+%{
+/*-
+ * Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua>
+ *		at Electronni Visti IA, Kiev, Ukraine.
+ *			All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <ctype.h>
+#include <err.h>
+#include <limits.h>
+#include <unistd.h>
+#include <string.h>
+#include <sysexits.h>
+#include "common.h"
+#include "y.tab.h"
+
+int line_no = 1, save_no, fromsubs;
+u_char buf[BUFSIZE], *ptr;
+FILE *map_fp;
+YY_BUFFER_STATE main_buf, map_buf;
+#ifdef FLEX_DEBUG
+YYSTYPE yylval;
+#endif /* FLEX_DEBUG */
+int yylex(void);
+%}
+%%
+<INITIAL,charmap,nchar,subs,subs2>[ \t]+      ;
+<subs2>\"               { ptr = buf; BEGIN(string); }
+<subs>\<                { ptr = buf; fromsubs = 1; BEGIN(name); }
+<INITIAL>\<             { ptr = buf; fromsubs = 0; BEGIN(name); }
+^#.*\n			line_no++;
+^\n			line_no++;
+<INITIAL>\\\n           line_no++;
+<INITIAL,nchar,subs>\\t      { yylval.ch = '\t'; return CHAR; }
+<INITIAL,nchar,subs>\\n      { yylval.ch = '\n'; return CHAR; }
+<INITIAL,nchar,subs>\\b      { yylval.ch = '\b'; return CHAR; }
+<INITIAL,nchar,subs>\\f      { yylval.ch = '\f'; return CHAR; }
+<INITIAL,nchar,subs>\\v      { yylval.ch = '\v'; return CHAR; }
+<INITIAL,nchar,subs>\\r      { yylval.ch = '\r'; return CHAR; }
+<INITIAL,nchar,subs>\\a      { yylval.ch = '\a'; return CHAR; }
+<subs2>\n               {
+	line_no++;
+	BEGIN(INITIAL);
+	return '\n';
+}
+<INITIAL,nchar>\n       {
+	line_no++;
+	if (map_fp != NULL) {
+		ptr = buf;
+		BEGIN(defn);
+	}
+	return '\n';
+}
+<INITIAL>[;,{}()]       return *yytext;
+<INITIAL>substitute     { BEGIN(subs); return SUBSTITUTE; }
+<subs>with              { BEGIN(subs2); return WITH; }
+<INITIAL>order          return ORDER;
+<INITIAL>charmap        BEGIN(charmap);
+<INITIAL>;[ \t]*\.\.\.[ \t]*;   return RANGE;
+<INITIAL,nchar,subs>\\[0-7]{3}       {
+	u_int v;
+
+	sscanf(&yytext[1], "%o", &v);
+	yylval.ch = (u_char)v;
+	return CHAR;
+}
+<INITIAL,nchar,subs>\\x[0-9a-fA-F]{2}   {
+	u_int v;
+
+	sscanf(&yytext[2], "%x", &v);
+	yylval.ch = (u_char)v;
+	return CHAR;
+}
+<INITIAL,nchar,subs>\\. { yylval.ch = yytext[1]; return CHAR; }
+<INITIAL,nchar,subs>.   { yylval.ch = *yytext; return CHAR; }
+<defn>^#.*\n            line_no++;
+<defn>[ \t]+            {
+	if (ptr == buf)
+		errx(EX_UNAVAILABLE, "map expected near line %u of %s",
+		     line_no, map_name);
+	*ptr = '\0';
+	strcpy(yylval.str, buf);
+	BEGIN(nchar);
+	return DEFN;
+}
+<name>\/\/              {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "name buffer overflow near line %u, character '/'",
+		     line_no);
+	*ptr++ = '/';
+}
+<name>\/\>              {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "name buffer overflow near line %u, character '>'",
+		     line_no);
+	*ptr++ = '>';
+}
+<string>\\\"		{
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\"'",
+		     line_no);
+	*ptr++ = '"';
+}
+<name>\>		{
+	u_int i;
+
+	if (ptr == buf)
+		errx(EX_UNAVAILABLE, "non-empty name expected near line %u",
+		     line_no);
+	*ptr = '\0';
+	for (i = 0; i <= UCHAR_MAX; i++) {
+		if (strcmp(charmap_table[i], buf) == 0)
+			goto findit;
+	}
+	errx(EX_UNAVAILABLE, "name <%s> not 'charmap'-defined near line %u",
+		buf, line_no);
+ findit:
+	yylval.ch = i;
+	if (fromsubs)
+		BEGIN(subs);
+	else
+		BEGIN(INITIAL);
+	return CHAR;
+}
+<string>\"		{
+	*ptr = '\0';
+	strcpy(yylval.str, buf);
+	BEGIN(subs2);
+	return STRING;
+}
+<name,defn>.            {
+	const char *s = (map_fp != NULL) ? map_name : "input";
+
+	if (!isascii(*yytext) || !isprint(*yytext))
+		errx(EX_UNAVAILABLE, "non-ASCII or non-printable character 0x%02x not allowed in the map/name near line %u of %s",
+		     *yytext, line_no, s);
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "map/name buffer overflow near line %u of %s, character '%c'",
+		     line_no, s, *yytext);
+	*ptr++ = *yytext;
+}
+<string>\\t             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\t'",
+		     line_no);
+	*ptr++ = '\t';
+}
+<string>\\b             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\b'",
+		     line_no);
+	*ptr++ = '\b';
+}
+<string>\\f             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\f'",
+		     line_no);
+	*ptr++ = '\f';
+}
+<string>\\v             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\v'",
+		     line_no);
+	*ptr++ = '\v';
+}
+<string>\\n             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\n'",
+		     line_no);
+	*ptr++ = '\n';
+}
+<string>\\r             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\r'",
+		     line_no);
+	*ptr++ = '\r';
+}
+<string>\\a             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '\\a'",
+		     line_no);
+	*ptr++ = '\a';
+}
+<name,string,defn>\n            {
+	const char *s = (map_fp != NULL) ? map_name : "input";
+
+	errx(EX_UNAVAILABLE, "unterminated map/name/string near line %u of %s", line_no, s);
+}
+<name,string,nchar><<EOF>>      {
+	const char *s = (map_fp != NULL) ? map_name : "input";
+
+	errx(EX_UNAVAILABLE, "premature EOF in the name/string/char near line %u of %s", line_no, s);
+}
+<string>\\x[0-9a-f]{2}          {
+	u_int v;
+
+	sscanf(&yytext[2], "%x", &v);
+	*ptr++ = (u_char)v;
+}
+<string>\\[0-7]{3}              {
+	u_int v;
+
+	sscanf(&yytext[1], "%o", &v);
+	*ptr++ = (u_char)v;
+}
+<string>\\.             {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '%c'",
+		     line_no, yytext[1]);
+	*ptr++ = yytext[1];
+}
+<string>.               {
+	if(ptr >= buf + sizeof(buf) - 1)
+		errx(EX_UNAVAILABLE, "string buffer overflow near line %u, character '%c'",
+		     line_no, *yytext);
+	*ptr++ = *yytext;
+}
+<charmap>[^ \t\n]+	{
+	strcat(map_name, "/");
+	strcat(map_name, yytext);
+	if((map_fp = fopen(map_name, "r")) == NULL)
+		err(EX_UNAVAILABLE, "can't open 'charmap' file %s",
+		    map_name);
+	save_no = line_no;
+	line_no = 1;
+	map_buf = yy_new_buffer(map_fp, YY_BUF_SIZE);
+	main_buf = YY_CURRENT_BUFFER;
+	yy_switch_to_buffer(map_buf);
+	ptr = buf;
+	BEGIN(defn);
+}
+<charmap>\n             {
+	errx(EX_UNAVAILABLE, "'charmap' file name expected near line %u",
+	     line_no);
+}
+<charmap><<EOF>>        {
+	errx(EX_UNAVAILABLE, "'charmap' file name expected near line %u",
+	     line_no);
+}
+<INITIAL,defn><<EOF>>                 {
+	if(map_fp != NULL) {
+		if (ptr != buf)
+			errx(EX_UNAVAILABLE, "premature EOF in the map near line %u of %s", line_no, map_name);
+		yy_switch_to_buffer(main_buf);
+		yy_delete_buffer(map_buf);
+		fclose(map_fp);
+		map_fp = NULL;
+		line_no = save_no;
+		BEGIN(INITIAL);
+	} else
+		yyterminate();
+}
+%%
+#ifdef FLEX_DEBUG
+main()
+{
+	while(yylex())
+		;
+	return 0;
+}
+#endif /* FLEX_DEBUG */