Reviewed by: JKH

Submitted by:	Wes Peters

Added strtok_r (reentrant) function and man page.

2 files changed, 217 insertions, 87 deletions

diff --git a/lib/libc/string/strtok.3 b/lib/libc/string/strtok.3
index 56db2b0..2dfb4da 100644
--- a/lib/libc/string/strtok.3
+++ b/lib/libc/string/strtok.3
@@ -1,3 +1,8 @@
+.\" Copyright (c) 1998 Softweyr LLC.  All rights reserved.
+.\"
+.\" strtok_r, from Berkeley strtok
+.\" Oct 13, 1998 by Wes Peters <wes@softweyr.com>
+.\"
 .\" Copyright (c) 1988, 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
 .\"
@@ -8,43 +13,54 @@
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
+.\"
 .\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\" 3. All advertising materials mentioning features or use of this software
-.\"    must display the following acknowledgement:
-.\"	This product includes software developed by the University of
-.\"	California, Berkeley and its contributors.
-.\" 4. Neither the name of the University nor the names of its contributors
-.\"    may be used to endorse or promote products derived from this software
-.\"    without specific prior written permission.
+.\"    notices, this list of conditions and the following disclaimer.
+.\" 
+.\" 2. Redistributions in binary form must reproduce the above
+.\"    copyright notices, this list of conditions and the following
+.\"    disclaimer in the documentation and/or other materials provided
+.\"    with the distribution.
+.\" 
+.\" 3. All advertising materials mentioning features or use of this
+.\"    software must display the following acknowledgement:
 .\"
-.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
-.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
-.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\"	This product includes software developed by Softweyr LLC, the
+.\"      University of California, Berkeley, and its contributors.
+.\"
+.\" 4. Neither the name of Softweyr LLC, the University nor the names
+.\"    of its contributors may be used to endorse or promote products
+.\"    derived from this software without specific prior written
+.\"    permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY SOFTWEYR LLC, THE REGENTS AND
+.\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+.\" DISCLAIMED.  IN NO EVENT SHALL SOFTWEYR LLC, THE REGENTS, OR
+.\" CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+.\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+.\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+.\" USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+.\" ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
 .\"     @(#)strtok.3	8.2 (Berkeley) 2/3/94
 .\"
-.Dd February 3, 1994
+.Dd November 27, 1998
 .Dt STRTOK 3
-.Os BSD 3
+.Os FreeBSD 3.0
 .Sh NAME
-.Nm strtok
+.Nm strtok, strtok_r
 .Nd string tokens
 .Sh SYNOPSIS
 .Fd #include <string.h>
 .Ft char *
 .Fn strtok "char *str" "const char *sep"
+.Ft char *
+.Fn strtok_r "char *str" "const char *sep" "char **last"
 .Sh DESCRIPTION
 .Bf -symbolic
 This interface is obsoleted by strsep(3).
@@ -69,13 +85,51 @@ The separator string,
 must be supplied each time, and may change between calls.
 .Pp
 The
+.Fn strtok_r
+function is a reentrant version of
+.Fn strtok ,
+the context pointer
+.Fa last
+must be provided on each call.
+.Fn strtok_r
+may also be used to nest two parsing loops within one another, as
+long as seperate context pointers are used.
+.Pp
+The
 .Fn strtok
-function
-returns a pointer to the beginning of each subsequent token in the string,
+and
+.Fn strtok_r
+functions
+return a pointer to the beginning of each subsequent token in the string,
 after replacing the token itself with a
 .Dv NUL
 character.
 When no more tokens remain, a null pointer is returned.
+.Sh EXAMPLE
+The following uses
+.Fn strtok_r ()
+to parse two strings using separate contexts:
+.Bd -literal
+char test[80], blah[80];
+char *sep = "\\/:;=-";
+char *word, *phrase, *brkt, *brkb;
+
+strcpy(test, "This;is.a:test:of=the/string\\tokenizer-function.");
+
+for (word = strtok_r(test, sep, &brkt);
+     word;
+     word = strtok_r(NULL, sep, &brkt))
+{
+    strcpy(blah, "blah:blat:blab:blag");
+
+    for (phrase = strtok_r(blah, sep, &brkb);
+         phrase;
+         phrase = strtok_r(NULL, sep, &brkb))
+    {
+        printf("So far we're at %s:%s\n", word, phrase);
+    }
+}
+.Ed
 .Sh SEE ALSO
 .Xr index 3 ,
 .Xr memchr 3 ,
@@ -94,8 +148,6 @@ function
 conforms to
 .St -ansiC .
 .Sh BUGS
-There is no way to get tokens from multiple strings simultaneously.
-.Pp
 The System V
 .Fn strtok ,
 if handed a string containing only delimiter characters,
@@ -108,3 +160,8 @@ value.
 Since this implementation always alters the next starting point,
 such a sequence of calls would always return
 .Dv NULL .
+.Sh AUTHOR
+Wes Peters, Softweyr LLC: 
+.Xr <wes@softweyr.com>
+.br
+Based on the FreeBSD 3.0 implementation.
diff --git a/lib/libc/string/strtok.c b/lib/libc/string/strtok.c
index 9151cf2..1140bfc 100644
--- a/lib/libc/string/strtok.c
+++ b/lib/libc/string/strtok.c
@@ -1,90 +1,163 @@
 /*
+ * Copyright (c) 1998 Softweyr LLC.  All rights reserved.
+ *
+ * strtok_r, from Berkeley strtok
+ * Oct 13, 1998 by Wes Peters <wes@softweyr.com>
+ *
  * Copyright (c) 1988, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
+ *
  * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
+ *    notices, this list of conditions and the following disclaimer.
+ * 
  * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
+ *    notices, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ * 
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
+ *
+ *	This product includes software developed by Softweyr LLC, the
+ *      University of California, Berkeley, and its contributors.
+ *
  * 4. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * THIS SOFTWARE IS PROVIDED BY SOFTWEYR LLC, THE REGENTS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL SOFTWEYR LLC, THE
+ * REGENTS, OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#if defined(LIBC_SCCS) && !defined(lint)
-static char sccsid[] = "@(#)strtok.c	8.1 (Berkeley) 6/4/93";
-#endif /* LIBC_SCCS and not lint */
-
 #include <stddef.h>
 #include <string.h>
 
 char *
-strtok(s, delim)
-	register char *s;
-	register const char *delim;
+strtok_r(char *s, const char *delim, char **last)
 {
-	register char *spanp;
-	register int c, sc;
-	char *tok;
-	static char *last;
+    char *spanp;
+    int c, sc;
+    char *tok;
 
+    if (s == NULL && (s = *last) == NULL)
+    {
+	return NULL;
+    }
 
-	if (s == NULL && (s = last) == NULL)
-		return (NULL);
-
-	/*
-	 * Skip (span) leading delimiters (s += strspn(s, delim), sort of).
-	 */
+    /*
+     * Skip (span) leading delimiters (s += strspn(s, delim), sort of).
+     */
 cont:
-	c = *s++;
-	for (spanp = (char *)delim; (sc = *spanp++) != 0;) {
-		if (c == sc)
-			goto cont;
+    c = *s++;
+    for (spanp = (char *)delim; (sc = *spanp++) != 0; )
+    {
+	if (c == sc)
+	{
+	    goto cont;
 	}
+    }
 
-	if (c == 0) {		/* no non-delimiter characters */
-		last = NULL;
-		return (NULL);
+    if (c == 0)		/* no non-delimiter characters */
+    {
+	*last = NULL;
+	return NULL;
+    }
+    tok = s - 1;
+
+    /*
+     * Scan token (scan for delimiters: s += strcspn(s, delim), sort of).
+     * Note that delim must have one NUL; we stop if we see that, too.
+     */
+    for (;;)
+    {
+	c = *s++;
+	spanp = (char *)delim;
+	do
+	{
+	    if ((sc = *spanp++) == c)
+	    {
+		if (c == 0)
+		{
+		    s = NULL;
+		}
+		else
+		{
+		    char *w = s - 1;
+		    *w = '\0';
+		}
+		*last = s;
+		return tok;
+	    }
 	}
-	tok = s - 1;
-
-	/*
-	 * Scan token (scan for delimiters: s += strcspn(s, delim), sort of).
-	 * Note that delim must have one NUL; we stop if we see that, too.
-	 */
-	for (;;) {
-		c = *s++;
-		spanp = (char *)delim;
-		do {
-			if ((sc = *spanp++) == c) {
-				if (c == 0)
-					s = NULL;
-				else
-					s[-1] = 0;
-				last = s;
-				return (tok);
-			}
-		} while (sc != 0);
+	while (sc != 0);
+    }
+    /* NOTREACHED */
+}
+
+
+char *
+strtok(char *s, const char *delim)
+{
+    static char *last;
+
+    return strtok_r(s, delim, &last);
+}
+
+
+#if defined(DEBUG_STRTOK)
+
+/*
+ * Test the tokenizer.
+ */
+int
+main()
+{
+    char test[80], blah[80];
+    char *sep = "\\/:;=-";
+    char *word, *phrase, *brkt, *brkb;
+
+    printf("String tokenizer test:\n");
+
+    strcpy(test, "This;is.a:test:of=the/string\\tokenizer-function.");
+
+    for (word = strtok(test, sep);
+	 word;
+	 word = strtok(NULL, sep))
+    {
+	printf("Next word is \"%s\".\n", word);
+    }
+
+    phrase = "foo";
+
+    strcpy(test, "This;is.a:test:of=the/string\\tokenizer-function.");
+
+    for (word = strtok_r(test, sep, &brkt);
+	 word;
+	 word = strtok_r(NULL, sep, &brkt))
+    {
+	strcpy(blah, "blah:blat:blab:blag");
+
+	for (phrase = strtok_r(blah, sep, &brkb);
+	     phrase;
+	     phrase = strtok_r(NULL, sep, &brkb))
+	{
+	    printf("So far we're at %s:%s\n", word, phrase);
 	}
-	/* NOTREACHED */
+    }
+
+    return 0;
 }
+
+#endif /* DEBUG_STRTOK */