id,summary,reporter,owner,description,type,status,milestone,component,version,severity,resolution,keywords,cc
943,tokenizer fails with wchar_t,nobody,jbandela,"{{{
Hi,

The tokenizer fails when instantiated with wchar_t; there are a couple of
problems with the char_delimiters_separator ctor:
1. The third parameter is typed ""char"" rather than ""Char""
2. The initializers of returnable_ and nonreturnable_ are using """"
    which is of type ""char const *"" rather than ""Char const *"".
A patch is appended below.

Ram (amberarrow@yahoo.com)

====================================================

*** token_functions.old.hpp	Fri Jul  6 14:01:45 2001
--- token_functions.hpp	Mon Jul 30 15:34:34 2001
***************
*** 249,258 ****
    public:
  
       explicit char_delimiters_separator(bool return_delims = false, 
! 		 const Char * returnable = 0,const char * nonreturnable = 0)
! 		 :nonreturnable_(nonreturnable?nonreturnable:""""),
! 		 returnable_(returnable?returnable:""""),no_ispunct_(returnable),
! 		 no_isspace_(nonreturnable),return_delims_(return_delims){}
  
       void reset(){}
  
--- 249,263 ----
    public:
  
       explicit char_delimiters_separator(bool return_delims = false, 
! 		 const Char * returnable = 0,const Char * nonreturnable = 0)
! 		 :no_ispunct_(returnable),
! 		 no_isspace_(nonreturnable),return_delims_(return_delims)
! 		  {
! 			  if (nonreturnable)
! 				  nonreturnable_ = nonreturnable;
! 			  if (returnable)
! 				  returnable_ = returnable;
! 		  }
  
       void reset(){}
  

}}}",Bugs,closed,,None,None,,None,,