2008-12-12 Craig Howland <howland@LGSInnovations.com>

* libc/include/wchar.h:  Added wcstok() declaration.
        * libc/string/wcstok.c: New file, implementing wcstok.
        * libc/string/Makefile.am: Add wcstok support.
        * libc/string/wcstrings.tex: Ditto.
        * libc/string/Makefile.in: Regenerated.

newlib/ChangeLog

@@ -1,3 +1,11 @@
+2008-12-12  Craig Howland <howland@LGSInnovations.com>
+
+	* libc/include/wchar.h:  Added wcstok() declaration.
+	* libc/string/wcstok.c: New file, implementing wcstok.
+	* libc/string/Makefile.am: Add wcstok support.
+	* libc/string/wcstrings.tex: Ditto.
+	* libc/string/Makefile.in: Regenerated.
+
 2008-12-12  Ralf Corsepius  <ralf.corsepius@rtems.org>
 
 	* libc/stdio/fputws.c: Fix documentation.

newlib/libc/include/wchar.h

@@ -73,6 +73,7 @@ wchar_t	*_EXFUN(wcspbrk, (const wchar_t *, const wchar_t *));
 wchar_t	*_EXFUN(wcsrchr, (const wchar_t *, wchar_t));
 size_t	_EXFUN(wcsspn, (const wchar_t *, const wchar_t *));
 wchar_t	*_EXFUN(wcsstr, (const wchar_t *, const wchar_t *));
+wchar_t	*_EXFUN(wcstok, (wchar_t *, const wchar_t *, wchar_t **));
 int	_EXFUN(wcswidth, (const wchar_t *, size_t));
 size_t	_EXFUN(wcsxfrm, (wchar_t *, const wchar_t *, size_t));
 int	_EXFUN(wcwidth, (const wchar_t));

newlib/libc/string/Makefile.am

@@ -62,6 +62,7 @@ GENERAL_SOURCES = \
 	wcsrchr.c \
 	wcsspn.c \
 	wcsstr.c \
+	wcstok.c \
 	wcswidth.c \
 	wcsxfrm.c \
 	wcwidth.c \
@@ -130,7 +131,7 @@ wcscat.def	wcschr.def	wcscmp.def	wcscoll.def \
 wcscpy.def	wcscspn.def	wcpcpy.def	wcpncpy.def \
 wcslcat.def	wcslcpy.def	wcslen.def	wcsncat.def \
 wcsncmp.def	wcsncpy.def	wcsnlen.def	wcspbrk.def \
-wcsrchr.def	wcsspn.def	wcsstr.def \
+wcsrchr.def	wcsspn.def	wcsstr.def 	wcstok.def  \
 wcswidth.def	wcsxfrm.def	wcwidth.def	wmemchr.def \
 wmemcmp.def	wmemcpy.def	wmemmove.def	wmemset.def \
 memmem.def

newlib/libc/string/Makefile.in

@@ -85,11 +85,11 @@ am__objects_1 = lib_a-bcopy.$(OBJEXT) lib_a-bzero.$(OBJEXT) \
 	lib_a-wcsncmp.$(OBJEXT) lib_a-wcsncpy.$(OBJEXT) \
 	lib_a-wcsnlen.$(OBJEXT) lib_a-wcspbrk.$(OBJEXT) \
 	lib_a-wcsrchr.$(OBJEXT) lib_a-wcsspn.$(OBJEXT) \
-	lib_a-wcsstr.$(OBJEXT) lib_a-wcswidth.$(OBJEXT) \
+	lib_a-wcsstr.$(OBJEXT) lib_a-wcstok.$(OBJEXT) \
-	lib_a-wcsxfrm.$(OBJEXT) lib_a-wcwidth.$(OBJEXT) \
+	lib_a-wcswidth.$(OBJEXT) lib_a-wcsxfrm.$(OBJEXT) \
-	lib_a-wmemchr.$(OBJEXT) lib_a-wmemcmp.$(OBJEXT) \
+	lib_a-wcwidth.$(OBJEXT) lib_a-wmemchr.$(OBJEXT) \
-	lib_a-wmemcpy.$(OBJEXT) lib_a-wmemmove.$(OBJEXT) \
+	lib_a-wmemcmp.$(OBJEXT) lib_a-wmemcpy.$(OBJEXT) \
-	lib_a-wmemset.$(OBJEXT)
+	lib_a-wmemmove.$(OBJEXT) lib_a-wmemset.$(OBJEXT)
 @ELIX_LEVEL_1_FALSE@am__objects_2 = lib_a-bcmp.$(OBJEXT) \
 @ELIX_LEVEL_1_FALSE@	lib_a-memccpy.$(OBJEXT) \
 @ELIX_LEVEL_1_FALSE@	lib_a-mempcpy.$(OBJEXT) \
@@ -116,9 +116,9 @@ am__objects_4 = bcopy.lo bzero.lo index.lo memchr.lo memcmp.lo \
 	strstr.lo swab.lo u_strerr.lo wcscat.lo wcschr.lo wcscmp.lo \
 	wcscoll.lo wcscpy.lo wcscspn.lo wcslcat.lo wcslcpy.lo \
 	wcslen.lo wcsncat.lo wcsncmp.lo wcsncpy.lo wcsnlen.lo \
-	wcspbrk.lo wcsrchr.lo wcsspn.lo wcsstr.lo wcswidth.lo \
+	wcspbrk.lo wcsrchr.lo wcsspn.lo wcsstr.lo wcstok.lo \
-	wcsxfrm.lo wcwidth.lo wmemchr.lo wmemcmp.lo wmemcpy.lo \
+	wcswidth.lo wcsxfrm.lo wcwidth.lo wmemchr.lo wmemcmp.lo \
-	wmemmove.lo wmemset.lo
+	wmemcpy.lo wmemmove.lo wmemset.lo
 @ELIX_LEVEL_1_FALSE@am__objects_5 = bcmp.lo memccpy.lo mempcpy.lo \
 @ELIX_LEVEL_1_FALSE@	stpcpy.lo stpncpy.lo strndup.lo \
 @ELIX_LEVEL_1_FALSE@	strcasestr.lo strndup_r.lo wcpcpy.lo \
@@ -371,6 +371,7 @@ GENERAL_SOURCES = \
 	wcsrchr.c \
 	wcsspn.c \
 	wcsstr.c \
+	wcstok.c \
 	wcswidth.c \
 	wcsxfrm.c \
 	wcwidth.c \
@@ -420,7 +421,7 @@ wcscat.def	wcschr.def	wcscmp.def	wcscoll.def \
 wcscpy.def	wcscspn.def	wcpcpy.def	wcpncpy.def \
 wcslcat.def	wcslcpy.def	wcslen.def	wcsncat.def \
 wcsncmp.def	wcsncpy.def	wcsnlen.def	wcspbrk.def \
-wcsrchr.def	wcsspn.def	wcsstr.def \
+wcsrchr.def	wcsspn.def	wcsstr.def 	wcstok.def  \
 wcswidth.def	wcsxfrm.def	wcwidth.def	wmemchr.def \
 wmemcmp.def	wmemcpy.def	wmemmove.def	wmemset.def \
 memmem.def
@@ -838,6 +839,12 @@ lib_a-wcsstr.o: wcsstr.c
 lib_a-wcsstr.obj: wcsstr.c
 	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-wcsstr.obj `if test -f 'wcsstr.c'; then $(CYGPATH_W) 'wcsstr.c'; else $(CYGPATH_W) '$(srcdir)/wcsstr.c'; fi`
 
+lib_a-wcstok.o: wcstok.c
+	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-wcstok.o `test -f 'wcstok.c' || echo '$(srcdir)/'`wcstok.c
+
+lib_a-wcstok.obj: wcstok.c
+	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-wcstok.obj `if test -f 'wcstok.c'; then $(CYGPATH_W) 'wcstok.c'; else $(CYGPATH_W) '$(srcdir)/wcstok.c'; fi`
+
 lib_a-wcswidth.o: wcswidth.c
 	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-wcswidth.o `test -f 'wcswidth.c' || echo '$(srcdir)/'`wcswidth.c
 

newlib/libc/string/wcstok.c

@@ -0,0 +1,181 @@
+/*
+FUNCTION
+	<<wcstok>>---get next token from a string
+
+INDEX
+	wcstok
+
+
+ANSI_SYNOPSIS
+	#include <wchar.h>
+      	wchar_t *wcstok(wchar_t *<[source]>, const wchar_t *<[delimiters]>,
+			wchar_t **<[lasts]>)
+
+TRAD_SYNOPSIS
+	#include <wchar.h>
+	wchar_t *wcstok(<[source]>, <[delimiters]>, <[lasts]>)
+	wchar_t *<[source]>;
+	wchar_t *<[delimiters]>;
+	wchar_t **<[lasts]>;
+
+DESCRIPTION
+	The <<wcstok>> function is the wide-character equivalent of the
+	<<strtok_r>> function (which in turn is the same as the <<strtok>>
+	function with an added argument to make it thread-safe).
+
+	The <<wcstok>> function is used to isolate (one at a time)
+	sequential tokens in a null-terminated wide-character string,
+	<<*<[source]>>>.  A token is defined as a substring not containing
+	any wide-characters from <<*<[delimiters]>>>.
+
+	The first time that <<wcstok>> is called, <<*<[source]>>> should
+	be specified with the wide-character string to be searched, and
+	<<*<[lasts]>>>--but not <<lasts>>, which must be non-NULL--may be
+	random; subsequent calls, wishing to obtain further tokens from
+	the same string, should pass a null pointer for <<*<[source]>>>
+	instead but must supply <<*<[lasts]>>> unchanged from the last
+	call.  The separator wide-character string, <<*<[delimiters]>>>,
+	must be supplied each time and may change between calls.
+	A pointer to placeholder <<*<[lasts]>>> must be supplied by
+	the caller, and is set each time as needed to save the state
+	by <<wcstok>>.	Every call to <<wcstok>> with <<*<[source]>>>
+	== <<NULL>> must pass the value of <<*<[lasts]>>> as last set
+	by <<wcstok>>.
+
+	The <<wcstok>> function returns a pointer to the beginning of each 
+	subsequent token in the string, after replacing the separator 
+	wide-character itself with a null wide-character.  When no more tokens
+	remain, a null pointer is returned.
+
+RETURNS
+	<<wcstok>> returns a pointer to the first wide character of a token, or
+	<<NULL>> if there is no token.
+
+NOTES
+	<<wcstok>> is thread-safe (unlike <<strtok>>, but like <<strtok_r>>).
+	<<wcstok>> writes into the string being searched.
+
+PORTABILITY
+<<wcstok>> is C99 and POSIX.1-2001.
+
+<<wcstok>> requires no supporting OS subroutines.
+
+QUICKREF
+	strtok ansi pure
+*/
+/* wcstok for Newlib created by adapting strtok_r, 2008.  */
+/*
+ * Copyright (c) 1988 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <wchar.h>
+
+wchar_t *
+_DEFUN (wcstok, (s, delim, lasts),
+	register wchar_t *s _AND
+	register const wchar_t *delim _AND
+	wchar_t **lasts)
+{
+	register const wchar_t *spanp;
+	register int c, sc;
+	wchar_t *tok;
+
+
+	if (s == NULL && (s = *lasts) == NULL)
+		return (NULL);
+
+	/*
+	 * Skip (span) leading delimiters (s += wcsspn(s, delim), sort of).
+	 */
+cont:
+	c = *s++;
+	for (spanp = delim; (sc = *spanp++) != L'\0';) {
+		if (c == sc)  goto cont;
+	}
+
+	if (c == L'\0') {		/* no non-delimiter characters */
+		*lasts = NULL;
+		return (NULL);
+	}
+	tok = s - 1;
+
+	/*
+	 * Scan token (scan for delimiters: s += wcscspn(s, delim), sort of).
+	 * Note that delim must have one NUL; we stop if we see that, too.
+	 */
+	for (;;) {
+		c = *s++;
+		spanp = delim;
+		do {
+			if ((sc = *spanp++) == c) {
+				if (c == L'\0')
+					s = NULL;
+				else
+					s[-1] = L'\0';
+				*lasts = s;
+				return (tok);
+			}
+		} while (sc != L'\0');
+	}
+	/* NOTREACHED */
+}
+ 
+/* The remainder of this file can serve as a regression test.  Compile
+ *  with -D_REGRESSION_TEST.  */
+#if defined(_REGRESSION_TEST)	/* [Test code:  example from C99 standard */
+#include <stdio.h>
+#include <wchar.h>
+ 
+/* example from C99 standard with minor additions to be a test */
+int
+main(void)
+{
+int  errs=0;
+static wchar_t str1[] = L"?a???b,,,#c";
+static wchar_t str2[] = L"\t \t";
+wchar_t *t, *ptr1, *ptr2;
+ 
+t = wcstok(str1, L"?", &ptr1); // t points to the token L"a"
+if(wcscmp(t,L"a")) errs++;
+t = wcstok(NULL, L",", &ptr1); // t points to the token L"??b"
+if(wcscmp(t,L"??b")) errs++;
+t = wcstok(str2, L" \t", &ptr2); // t is a null pointer
+if(t != NULL) errs++;
+t = wcstok(NULL, L"#,", &ptr1); // t points to the token L"c"
+if(wcscmp(t,L"c")) errs++;
+t = wcstok(NULL, L"?", &ptr1); // t is a null pointer
+if(t != NULL) errs++;
+ 
+printf("wcstok() test ");
+if(errs)  printf("FAILED %d test cases", errs);
+  else    printf("passed");
+printf(".\n");
+ 
+return(errs);
+}
+#endif /* defined(_REGRESSION_TEST) ] */

newlib/libc/string/wcstrings.tex

@@ -30,6 +30,7 @@ declarations are in @file{wchar.h}.
 * wcsrchr::     Reverse search for wide character in string
 * wcsspn::      Find initial match in wide-character string
 * wcsstr::      Find wide-character string segment
+* wcstok::      Tokenize wide-character string
 * wcswidth::    Number of column positions of a wide-character string
 * wcsxfrm::     Locale-specific wide-character string transformation
 * wcwidth::     Number of column positions of a wide-character code
@@ -107,6 +108,9 @@ declarations are in @file{wchar.h}.
 @page
 @include string/wcsstr.def
 
+@page
+@include string/wcstok.def
+
 @page
 @include string/wcswidth.def