Add an implementation of wcstok(), based on strtok_r().

3 files changed, 213 insertions, 3 deletions

diff --git a/lib/libc/string/Makefile.inc b/lib/libc/string/Makefile.inc
index 77d63a0..c612a94 100644
--- a/lib/libc/string/Makefile.inc
+++ b/lib/libc/string/Makefile.inc
@@ -14,8 +14,8 @@ MISRCS+=bcmp.c bcopy.c bzero.c ffs.c index.c memccpy.c memchr.c memcmp.c \
 	strpbrk.c strrchr.c strsep.c strsignal.c strspn.c strstr.c strtok.c \
 	strxfrm.c swab.c wcscat.c wcschr.c wcscmp.c wcscpy.c wcscspn.c \
 	wcslcat.c wcslcpy.c wcslen.c wcsncat.c wcsncmp.c wcsncpy.c wcspbrk.c \
-	wcsrchr.c wcsspn.c wcsstr.c wcswidth.c wmemchr.c wmemcmp.c wmemcpy.c \
-	wmemmove.c wmemset.c
+	wcsrchr.c wcsspn.c wcsstr.c wcstok.c wcswidth.c wmemchr.c wmemcmp.c \
+	wmemcpy.c wmemmove.c wmemset.c
 
 
 # machine-dependent string sources
@@ -28,7 +28,8 @@ MAN+=	bcmp.3 bcopy.3 bstring.3 bzero.3 ffs.3 index.3 memccpy.3 memchr.3 \
 	memcmp.3 memcpy.3 memmove.3 memset.3 rindex.3 strcasecmp.3 strcat.3 \
 	strchr.3 strcmp.3 strcoll.3 strcpy.3 strcspn.3 strdup.3 strerror.3 \
 	string.3 strlcpy.3 strlen.3 strmode.3 strpbrk.3 strrchr.3 strsep.3 \
-	strspn.3 strstr.3 strtok.3 strxfrm.3 swab.3 wcswidth.3 wmemchr.3
+	strspn.3 strstr.3 strtok.3 strxfrm.3 swab.3 wcstok.3 wcswidth.3 \
+	wmemchr.3
 
 MLINKS+=strcasecmp.3 strncasecmp.3
 MLINKS+=strcat.3 strncat.3
diff --git a/lib/libc/string/wcstok.3 b/lib/libc/string/wcstok.3
new file mode 100644
index 0000000..f33e165
--- /dev/null
+++ b/lib/libc/string/wcstok.3
@@ -0,0 +1,120 @@
+.\" Copyright (c) 1998 Softweyr LLC.  All rights reserved.
+.\"
+.\" strtok_r, from Berkeley strtok
+.\" Oct 13, 1998 by Wes Peters <wes@softweyr.com>
+.\"
+.\" Copyright (c) 1988, 1991, 1993
+.\"	The Regents of the University of California.  All rights reserved.
+.\"
+.\" This code is derived from software contributed to Berkeley by
+.\" the American National Standards Committee X3, on Information
+.\" Processing Systems.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notices, this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above
+.\"    copyright notices, this list of conditions and the following
+.\"    disclaimer in the documentation and/or other materials provided
+.\"    with the distribution.
+.\"
+.\" 3. All advertising materials mentioning features or use of this
+.\"    software must display the following acknowledgement:
+.\"
+.\"	This product includes software developed by Softweyr LLC, the
+.\"      University of California, Berkeley, and its contributors.
+.\"
+.\" 4. Neither the name of Softweyr LLC, the University nor the names
+.\"    of its contributors may be used to endorse or promote products
+.\"    derived from this software without specific prior written
+.\"    permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY SOFTWEYR LLC, THE REGENTS AND
+.\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+.\" DISCLAIMED.  IN NO EVENT SHALL SOFTWEYR LLC, THE REGENTS, OR
+.\" CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+.\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+.\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+.\" USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+.\" ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd September 7, 2002
+.Dt WCSTOK 3
+.Os
+.Sh NAME
+.Nm wcstok
+.Nd split wide-character string into tokens
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In wchar.h
+.Ft wchar_t *
+.Fn wcstok "wchar_t * restrict str" "const wchar_t * restrict sep" "wchar_t ** restrict last"
+.Sh DESCRIPTION
+The
+.Fn wcstok
+function
+is used to isolate sequential tokens in a null-terminated wide character
+string,
+.Fa str .
+These tokens are separated in the string by at least one of the
+characters in
+.Fa sep .
+The first time that
+.Fn strtok
+is called,
+.Fa str
+should be specified; subsequent calls, wishing to obtain further tokens
+from the same string, should pass a null pointer instead.
+The separator string,
+.Fa sep ,
+must be supplied each time, and may change between calls.
+The context pointer
+.Fa last
+must be provided on each call.
+.Pp
+The
+.Fn wcstok
+function is the wide character counterpart of the
+.Fn strtok_r
+function.
+.Sh RETURN VALUES
+The
+.Fn wcstok
+function
+returns a pointer to the beginning of each subsequent token in the string,
+after replacing the token itself with a null wide character (L'\e0').
+When no more tokens remain, a null pointer is returned.
+.Sh COMPATIBILITY
+Some early implementations of
+.Fn wcstok
+omit the
+context pointer argument,
+.Fa last ,
+and maintain state across calls in a static variable like
+.Fn strtok
+does.
+.Sh SEE ALSO
+.Xr wcschr 3 ,
+.Xr wcscspn 3 ,
+.Xr wcspbrk 3 ,
+.Xr wcsrchr 3 ,
+.Xr wcsspn 3 ,
+.Xr strtok 3
+.Sh STANDARDS
+The
+.Fn wcstok
+function
+conforms to
+.St -isoC-99 .
diff --git a/lib/libc/string/wcstok.c b/lib/libc/string/wcstok.c
new file mode 100644
index 0000000..5d899ee
--- /dev/null
+++ b/lib/libc/string/wcstok.c
@@ -0,0 +1,89 @@
+/*-
+ * Copyright (c) 1998 Softweyr LLC.  All rights reserved.
+ *
+ * strtok_r, from Berkeley strtok
+ * Oct 13, 1998 by Wes Peters <wes@softweyr.com>
+ *
+ * Copyright (c) 1988, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notices, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notices, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by Softweyr LLC, the
+ *      University of California, Berkeley, and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY SOFTWEYR LLC, THE REGENTS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL SOFTWEYR LLC, THE
+ * REGENTS, OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <wchar.h>
+
+wchar_t *
+wcstok(wchar_t * __restrict s, const wchar_t * __restrict delim,
+    wchar_t ** __restrict last)
+{
+	wchar_t *spanp, *tok;
+	wchar_t c, sc;
+
+	if (s == NULL && (s = *last) == NULL)
+		return (NULL);
+
+	/*
+	 * Skip (span) leading delimiters (s += wcsspn(s, delim), sort of).
+	 */
+cont:
+	c = *s++;
+	for (spanp = (wchar_t *)delim; (sc = *spanp++) != L'\0';) {
+		if (c == sc)
+			goto cont;
+	}
+
+	if (c == L'\0') {	/* no non-delimiter characters */
+		*last = NULL;
+		return (NULL);
+	}
+	tok = s - 1;
+
+	/*
+	 * Scan token (scan for delimiters: s += wcscspn(s, delim), sort of).
+	 * Note that delim must have one NUL; we stop if we see that, too.
+	 */
+	for (;;) {
+		c = *s++;
+		spanp = (wchar_t *)delim;
+		do {
+			if ((sc = *spanp++) == c) {
+				if (c == L'\0')
+					s = NULL;
+				else
+					s[-1] = L'\0';
+				*last = s;
+				return (tok);
+			}
+		} while (sc != L'\0');
+	}
+	/* NOTREACHED */
+}