- Support for multibyte charsets in LIBICONV.

- CD9660_ICONV, NTFS_ICONV and MSDOSFS_ICONV kernel options
(with corresponding modules).
- kiconv(3) for loadable charset conversion tables support.

Submitted by:	Ryuichiro Imura <imura@ryu16.org>

7 files changed, 197 insertions, 48 deletions

diff --git a/sys/fs/ntfs/ntfs.h b/sys/fs/ntfs/ntfs.h
index fba06c1..6e19029 100644
--- a/sys/fs/ntfs/ntfs.h
+++ b/sys/fs/ntfs/ntfs.h
@@ -254,6 +254,8 @@ struct ntfsmount {
 	int		ntm_adnum;
  	wchar *		ntm_82u;	/* 8bit to Unicode */
  	char **		ntm_u28;	/* Unicode to 8 bit */
+	void *		ntm_ic_l2u;	/* Local to Unicode (iconv) */
+	void *		ntm_ic_u2l;	/* Unicode to Local (iconv) */
 };
 
 #define ntm_mftcn	ntm_bootfile.bf_mftcn
diff --git a/sys/fs/ntfs/ntfs_iconv.c b/sys/fs/ntfs/ntfs_iconv.c
new file mode 100644
index 0000000..347306e
--- /dev/null
+++ b/sys/fs/ntfs/ntfs_iconv.c
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2003 Ryuichiro Imura
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/mount.h>
+#include <sys/iconv.h>
+
+VFS_DECLARE_ICONV(ntfs);
diff --git a/sys/fs/ntfs/ntfs_subr.c b/sys/fs/ntfs/ntfs_subr.c
index cf2879c..aff97a3 100644
--- a/sys/fs/ntfs/ntfs_subr.c
+++ b/sys/fs/ntfs/ntfs_subr.c
@@ -40,6 +40,7 @@
 #include <sys/file.h>
 #include <sys/malloc.h>
 #include <sys/lock.h>
+#include <sys/iconv.h>
 
 /* #define NTFS_DEBUG 1 */
 #include <fs/ntfs/ntfs.h>
@@ -67,6 +68,8 @@ static wchar *ntfs_toupper_tab;
 static struct lock ntfs_toupper_lock;
 static signed int ntfs_toupper_usecount;
 
+struct iconv_functions *ntfs_iconv = NULL;
+
 /* support macro for ntfs_ntvattrget() */
 #define NTFS_AALPCMP(aalp,type,name,namelen) (				\
   (aalp->al_type == type) && (aalp->al_namelen == namelen) &&		\
@@ -665,20 +668,41 @@ ntfs_uastricmp(ntmp, ustr, ustrlen, astr, astrlen)
 	const char *astr;
 	size_t astrlen;
 {
-	size_t             i;
-	int             res;
+	int len;
+	size_t i, j, mbstrlen = astrlen;
+	int res;
+	wchar wc;
+
+	if (ntmp->ntm_ic_l2u) {
+		for (i = 0, j = 0; i < ustrlen && j < astrlen; i++, j++) {
+			if (j < astrlen -1) {
+				wc = (wchar)astr[j]<<8 | (astr[j+1]&0xFF);
+				len = 2;
+			} else {
+				wc = (wchar)astr[j]<<8 & 0xFF00;
+				len = 1;
+			}
+			res = ((int) NTFS_TOUPPER(ustr[i])) -
+				((int)NTFS_TOUPPER(NTFS_82U(wc, &len)));
+			j += len - 1;
+			mbstrlen -= len - 1;
 
-	/*
-	 * XXX We use NTFS_82U(NTFS_U28(c)) to get rid of unicode
-	 * symbols not covered by translation table
-	 */
-	for (i = 0; i < ustrlen && i < astrlen; i++) {
-		res = ((int) NTFS_TOUPPER(NTFS_82U(NTFS_U28(ustr[i])))) -
-			((int)NTFS_TOUPPER(NTFS_82U(astr[i])));
-		if (res)
-			return res;
+			if (res)
+				return res;
+		}
+	} else {
+		/*
+		 * We use NTFS_82U(NTFS_U28(c)) to get rid of unicode
+		 * symbols not covered by translation table
+		 */
+		for (i = 0; i < ustrlen && i < astrlen; i++) {
+			res = ((int) NTFS_TOUPPER(NTFS_82U(NTFS_U28(ustr[i]), &len))) -
+				((int)NTFS_TOUPPER(NTFS_82U((wchar)astr[i], &len)));
+			if (res)
+				return res;
+		}
 	}
-	return (ustrlen - astrlen);
+	return (ustrlen - mbstrlen);
 }
 
 /*
@@ -692,15 +716,25 @@ ntfs_uastrcmp(ntmp, ustr, ustrlen, astr, astrlen)
 	const char *astr;
 	size_t astrlen;
 {
-	size_t             i;
-	int             res;
-
-	for (i = 0; (i < ustrlen) && (i < astrlen); i++) {
-		res = (int) (((char)NTFS_U28(ustr[i])) - astr[i]);
+	char u, l;
+	size_t i, j, mbstrlen = astrlen;
+	int res;
+	wchar wc;
+
+	for (i = 0, j = 0; (i < ustrlen) && (j < astrlen); i++, j++) {
+		res = 0;
+		wc = NTFS_U28(ustr[i]);
+		u = (char)(wc>>8);
+		l = (char)wc;
+		if (u != '\0' && j < astrlen -1) {
+			res = (int) (u - astr[j++]);
+			mbstrlen--;
+		}
+		res = (res<<8) + (int) (l - astr[j]);
 		if (res)
 			return res;
 	}
-	return (ustrlen - astrlen);
+	return (ustrlen - mbstrlen);
 }
 
 /* 
@@ -2003,11 +2037,18 @@ ntfs_toupper_unuse()
 int
 ntfs_u28_init(
 	struct ntfsmount *ntmp,
-	wchar *u2w)
+	wchar *u2w,
+	char *cs_local,
+	char *cs_ntfs)
 {
 	char ** u28;
 	int i, j, h, l;
 
+	if (ntfs_iconv && cs_local) {
+		ntfs_iconv->open(cs_local, cs_ntfs, &ntmp->ntm_ic_u2l);
+		return (0);
+	}
+
 	MALLOC(u28, char **, 256 * sizeof(char*), M_TEMP, M_WAITOK | M_ZERO);
 
 	for (i=0; i<256; i++) {
@@ -2034,8 +2075,12 @@ ntfs_u28_uninit(struct ntfsmount *ntmp)
 	char ** u28;
 	int i;
 
-	if (ntmp->ntm_u28 == NULL)
+	if (ntmp->ntm_u28 == NULL) {
+		if (ntfs_iconv && ntmp->ntm_ic_u2l) {
+			ntfs_iconv->close(ntmp->ntm_ic_u2l);
+		}
 		return (0);
+	}
 
 	u28 = ntmp->ntm_u28;
 
@@ -2051,22 +2096,21 @@ ntfs_u28_uninit(struct ntfsmount *ntmp)
 int
 ntfs_82u_init(
 	struct ntfsmount *ntmp,
-	u_int16_t *u2w)
+	char *cs_local,
+	char *cs_ntfs)
 {
 	wchar * _82u;
 	int i;
 
+	if (ntfs_iconv && cs_local) {
+		ntfs_iconv->open(cs_ntfs, cs_local, &ntmp->ntm_ic_l2u);
+		return (0);
+	}
+
 	MALLOC(_82u, wchar *, 256 * sizeof(wchar), M_TEMP, M_WAITOK);
 
-	if (u2w == NULL) {
-		for (i=0; i<256; i++)
-			_82u[i] = i;
-	} else {
-		for (i=0; i<128; i++)
+	for (i=0; i<256; i++)
 			_82u[i] = i;
-		for (i=0; i<128; i++)
-			_82u[i+128] = u2w[i];
-	}
 
 	ntmp->ntm_82u = _82u;
 
@@ -2076,6 +2120,14 @@ ntfs_82u_init(
 int
 ntfs_82u_uninit(struct ntfsmount *ntmp)
 {
+
+	if (ntmp->ntm_82u == NULL) {
+		if (ntfs_iconv && ntmp->ntm_ic_l2u) {
+			ntfs_iconv->close(ntmp->ntm_ic_l2u);
+		}
+		return (0);
+	}
+
 	FREE(ntmp->ntm_82u, M_TEMP);
 	return (0);
 }
@@ -2086,12 +2138,31 @@ ntfs_82u_uninit(struct ntfsmount *ntmp)
  * and substitutes a '_' for it if the result would be '\0';
  * something better has to be definitely though out
  */
-char
+wchar
 ntfs_u28(
 	struct ntfsmount *ntmp, 
 	wchar wc)
 {
-	char * p;
+	char *p, *outp, inbuf[3], outbuf[3];
+	size_t ilen, olen;
+
+	if (ntfs_iconv && ntmp->ntm_ic_u2l) {
+		ilen = olen = 2;
+
+		inbuf[0] = (char)(wc>>8);
+		inbuf[1] = (char)wc;
+		inbuf[2] = '\0';
+		p = inbuf;
+		outp = outbuf;
+		ntfs_iconv->convchr(ntmp->ntm_ic_u2l, (const char **)&p, &ilen,
+				    &outp, &olen);
+		if (olen == 1) {
+			return ((wchar)(outbuf[0]));
+		} else if (olen == 0) {
+			return ((wchar)((outbuf[0]<<8) | (outbuf[1]&0xFF)));
+		}
+		return ('?');
+	}
 
 	p = ntmp->ntm_u28[(wc>>8)&0xFF];
 	if (p == NULL)
@@ -2099,3 +2170,36 @@ ntfs_u28(
 	return (p[wc&0xFF]);
 }
 
+wchar
+ntfs_82u(
+	struct ntfsmount *ntmp, 
+	wchar wc,
+	int *len)
+{
+	char *p, *outp, inbuf[3], outbuf[3];
+	wchar uc;
+	size_t ilen, olen;
+
+	if (ntfs_iconv && ntmp->ntm_ic_l2u) {
+		ilen = (size_t)*len;
+		olen = 2;
+
+		inbuf[0] = (char)(wc>>8);
+		inbuf[1] = (char)wc;
+		inbuf[2] = '\0';
+		p = inbuf;
+		outp = outbuf;
+		ntfs_iconv->convchr(ntmp->ntm_ic_l2u, (const char **)&p, &ilen,
+				    &outp, &olen);
+		*len -= (int)ilen;
+		uc = (wchar)((outbuf[0]<<8) | (outbuf[1]&0xFF));
+
+		return (uc);
+	}
+
+	if (ntmp->ntm_82u != NULL)
+		return (ntmp->ntm_82u[wc&0xFF]);
+
+	return ('?');
+}
+
diff --git a/sys/fs/ntfs/ntfs_subr.h b/sys/fs/ntfs/ntfs_subr.h
index 1d676ab..c0ecc75 100644
--- a/sys/fs/ntfs/ntfs_subr.h
+++ b/sys/fs/ntfs/ntfs_subr.h
@@ -108,13 +108,14 @@ void ntfs_toupper_unuse(void);
 int ntfs_fget(struct ntfsmount *, struct ntnode *, int, char *, struct fnode **);
 void ntfs_frele(struct fnode *);
 
-int ntfs_u28_init(struct ntfsmount *ntmp, wchar *u2w);
+int ntfs_u28_init(struct ntfsmount *ntmp, wchar *u2w, char *cs_local, char *cs_ntfs);
 int ntfs_u28_uninit(struct ntfsmount *ntmp);
-int ntfs_82u_init(struct ntfsmount *ntmp, u_int16_t *u2w);
+int ntfs_82u_init(struct ntfsmount *ntmp, char *cs_local, char *cs_ntfs);
 int ntfs_82u_uninit(struct ntfsmount *ntmp);
-char ntfs_u28(struct ntfsmount *ntmp, wchar wc);
+wchar ntfs_u28(struct ntfsmount *ntmp, wchar wc);
+wchar ntfs_82u(struct ntfsmount *ntmp, wchar wc, int *len);
 #define NTFS_U28(ch)		ntfs_u28(ntmp, (ch))
-#define NTFS_82U(ch)		(ntmp->ntm_82u[(ch)&0xFF])
+#define NTFS_82U(ch, len)	ntfs_82u(ntmp, (ch), len)
 #define	NTFS_UASTRCMP(ustr, ustrlen, astr, astrlen)	\
 	ntfs_uastrcmp(ntmp, (ustr), (ustrlen), (astr), (astrlen))
 #define	NTFS_UASTRICMP(ustr, ustrlen, astr, astrlen)	\
diff --git a/sys/fs/ntfs/ntfs_vfsops.c b/sys/fs/ntfs/ntfs_vfsops.c
index cd5e062..a959e8c 100644
--- a/sys/fs/ntfs/ntfs_vfsops.c
+++ b/sys/fs/ntfs/ntfs_vfsops.c
@@ -350,15 +350,15 @@ ntfs_mountfs(devvp, mp, argsp, td)
 	ntmp->ntm_flag = argsp->flag;
 
 	/* Copy in the 8-bit to Unicode conversion table */
-	if (argsp->flag & NTFSMNT_U2WTABLE) {
-		ntfs_82u_init(ntmp, argsp->u2w);
+	/* Initialize Unicode to 8-bit table from 8toU table */
+	if (argsp->flag & NTFS_MFLAG_KICONV) {
+		ntfs_82u_init(ntmp, argsp->cs_local, argsp->cs_ntfs);
+		ntfs_u28_init(ntmp, NULL, argsp->cs_local, argsp->cs_ntfs);
 	} else {
-		ntfs_82u_init(ntmp, NULL);
+		ntfs_82u_init(ntmp, NULL, NULL);
+		ntfs_u28_init(ntmp, ntmp->ntm_82u, NULL, NULL);
 	}
 
-	/* Initialize Unicode to 8-bit table from 8toU table */
-	ntfs_u28_init(ntmp, ntmp->ntm_82u);
-
 	mp->mnt_data = (qaddr_t)ntmp;
 
 	dprintf(("ntfs_mountfs(): case-%s,%s uid: %d, gid: %d, mode: %o\n",
@@ -794,3 +794,4 @@ static struct vfsops ntfs_vfsops = {
 	.vfs_vptofh =	ntfs_vptofh,
 };
 VFS_SET(ntfs_vfsops, ntfs, 0);
+MODULE_VERSION(ntfs, 1);
diff --git a/sys/fs/ntfs/ntfs_vnops.c b/sys/fs/ntfs/ntfs_vnops.c
index c24c9bf..4333c36 100644
--- a/sys/fs/ntfs/ntfs_vnops.c
+++ b/sys/fs/ntfs/ntfs_vnops.c
@@ -495,7 +495,8 @@ ntfs_readdir(ap)
 	register struct ntnode *ip = FTONT(fp);
 	struct uio *uio = ap->a_uio;
 	struct ntfsmount *ntmp = ip->i_mp;
-	int i, error = 0;
+	int i, j, error = 0;
+	wchar c;
 	u_int32_t faked = 0, num;
 	int ncookies = 0;
 	struct dirent cde;
@@ -552,14 +553,17 @@ ntfs_readdir(ap)
 			if(!ntfs_isnamepermitted(ntmp,iep))
 				continue;
 
-			for(i=0; i<iep->ie_fnamelen; i++) {
-				cde.d_name[i] = NTFS_U28(iep->ie_fname[i]);
+			for(i=0, j=0; i<iep->ie_fnamelen; i++, j++) {
+				c = NTFS_U28(iep->ie_fname[i]);
+				if (c&0xFF00)
+					cde.d_name[j++] = (char)(c>>8);
+				cde.d_name[j] = (char)c&0xFF;
 			}
-			cde.d_name[i] = '\0';
+			cde.d_name[j] = '\0';
 			dprintf(("ntfs_readdir: elem: %d, fname:[%s] type: %d, flag: %d, ",
 				num, cde.d_name, iep->ie_fnametype,
 				iep->ie_flag));
-			cde.d_namlen = iep->ie_fnamelen;
+			cde.d_namlen = j;
 			cde.d_fileno = iep->ie_number;
 			cde.d_type = (iep->ie_fflag & NTFS_FFLAG_DIR) ? DT_DIR : DT_REG;
 			cde.d_reclen = sizeof(struct dirent);
diff --git a/sys/fs/ntfs/ntfsmount.h b/sys/fs/ntfs/ntfsmount.h
index 0c143ed..5497ad9 100644
--- a/sys/fs/ntfs/ntfsmount.h
+++ b/sys/fs/ntfs/ntfsmount.h
@@ -30,7 +30,7 @@
 
 #define	NTFS_MFLAG_CASEINS	0x00000001
 #define	NTFS_MFLAG_ALLNAMES	0x00000002
-#define	NTFSMNT_U2WTABLE	0x00000004
+#define	NTFS_MFLAG_KICONV	0x00000004
 
 struct ntfs_args {
 	char	*fspec;			/* block special device to mount */
@@ -39,5 +39,6 @@ struct ntfs_args {
 	gid_t	gid;			/* gid that owns ntfs files */
 	mode_t	mode;			/* mask to be applied for ntfs perms */
 	u_long	flag;			/* additional flags */
-	u_int16_t u2w[256];		/* Unix to Wchar */
+	char	*cs_ntfs;		/* NTFS Charset */
+	char	*cs_local;		/* Local Charset */
 };