43a44,45
> #include <ctype.h>	// required for wide version of character checkers
> 
50a53,55
> #ifdef iswpunct
> #  undef iswpunct
> #endif
53a59,61
> #ifdef iswspace
> #  undef iswspace
> #endif
60a69,70
>  using ::iswpunct;
>  using ::iswspace;
65d74
< 
197a207,227
>   //===========================================================================
> 	// Tokenizer was broken for wide character separators, at least on Windows, since
> 	// CRT functions isspace etc are only expect values in [0, 0xFF]. Debug build asserts
> 	// if higher values are passed in. The traits extension class should take care of this.
> 	// Assuming that the conditional will always get optimized out in the function
> 	// implementations, argument types are not a problem since both forms of character classifiers
> 	// expect an int.
> 	template<typename traits>
> 	struct traits_extension : public traits {
> 		using typename traits::char_type;
> 
> 		static bool isspace(char_type c)
> 		{
> 			return sizeof(char_type) == 1 ? std::isspace(c) : std::iswspace(c);
> 		}
> 
> 		static bool ispunct(char_type c)
> 		{
> 			return sizeof(char_type) == 1 ? std::ispunct(c) : std::iswpunct(c);
> 		}
> 	};
383c413
<     typename Traits = typename std::basic_string<Char>::traits_type >
---
>     typename Tr = typename std::basic_string<Char>::traits_type >
386c416
<     typename Traits = std::basic_string<Char>::traits_type >
---
>     typename Tr = std::basic_string<Char>::traits_type >
389a420
> 	typedef tokenizer_detail::traits_extension<Tr> Traits;
502c533
<         return std::ispunct(E) != 0;
---
>         return Traits::ispunct(E) != 0;
511c542
<         return std::isspace(E) != 0;
---
>         return Traits::isspace(E) != 0;
530c561
<     class Traits = typename std::basic_string<Char>::traits_type >
---
>     class Tr = typename std::basic_string<Char>::traits_type >
533c564
<     class Traits = std::basic_string<Char>::traits_type >
---
>     class Tr = std::basic_string<Char>::traits_type >
537a569
> 	typedef tokenizer_detail::traits_extension<Tr> Traits;
552c584
<           int r = std::ispunct(E);
---
>           int r = Traits::ispunct(E);
564c596
<           int r = std::isspace(E);
---
>           int r = Traits::isspace(E);