| 1 | 43a44,45
|
|---|
| 2 | > #include <ctype.h> // required for wide version of character checkers
|
|---|
| 3 | >
|
|---|
| 4 | 50a53,55
|
|---|
| 5 | > #ifdef iswpunct
|
|---|
| 6 | > # undef iswpunct
|
|---|
| 7 | > #endif
|
|---|
| 8 | 53a59,61
|
|---|
| 9 | > #ifdef iswspace
|
|---|
| 10 | > # undef iswspace
|
|---|
| 11 | > #endif
|
|---|
| 12 | 60a69,70
|
|---|
| 13 | > using ::iswpunct;
|
|---|
| 14 | > using ::iswspace;
|
|---|
| 15 | 65d74
|
|---|
| 16 | <
|
|---|
| 17 | 197a207,227
|
|---|
| 18 | > //===========================================================================
|
|---|
| 19 | > // Tokenizer was broken for wide character separators, at least on Windows, since
|
|---|
| 20 | > // CRT functions isspace etc are only expect values in [0, 0xFF]. Debug build asserts
|
|---|
| 21 | > // if higher values are passed in. The traits extension class should take care of this.
|
|---|
| 22 | > // Assuming that the conditional will always get optimized out in the function
|
|---|
| 23 | > // implementations, argument types are not a problem since both forms of character classifiers
|
|---|
| 24 | > // expect an int.
|
|---|
| 25 | > template<typename traits>
|
|---|
| 26 | > struct traits_extension : public traits {
|
|---|
| 27 | > using typename traits::char_type;
|
|---|
| 28 | >
|
|---|
| 29 | > static bool isspace(char_type c)
|
|---|
| 30 | > {
|
|---|
| 31 | > return sizeof(char_type) == 1 ? std::isspace(c) : std::iswspace(c);
|
|---|
| 32 | > }
|
|---|
| 33 | >
|
|---|
| 34 | > static bool ispunct(char_type c)
|
|---|
| 35 | > {
|
|---|
| 36 | > return sizeof(char_type) == 1 ? std::ispunct(c) : std::iswpunct(c);
|
|---|
| 37 | > }
|
|---|
| 38 | > };
|
|---|
| 39 | 383c413
|
|---|
| 40 | < typename Traits = typename std::basic_string<Char>::traits_type >
|
|---|
| 41 | ---
|
|---|
| 42 | > typename Tr = typename std::basic_string<Char>::traits_type >
|
|---|
| 43 | 386c416
|
|---|
| 44 | < typename Traits = std::basic_string<Char>::traits_type >
|
|---|
| 45 | ---
|
|---|
| 46 | > typename Tr = std::basic_string<Char>::traits_type >
|
|---|
| 47 | 389a420
|
|---|
| 48 | > typedef tokenizer_detail::traits_extension<Tr> Traits;
|
|---|
| 49 | 502c533
|
|---|
| 50 | < return std::ispunct(E) != 0;
|
|---|
| 51 | ---
|
|---|
| 52 | > return Traits::ispunct(E) != 0;
|
|---|
| 53 | 511c542
|
|---|
| 54 | < return std::isspace(E) != 0;
|
|---|
| 55 | ---
|
|---|
| 56 | > return Traits::isspace(E) != 0;
|
|---|
| 57 | 530c561
|
|---|
| 58 | < class Traits = typename std::basic_string<Char>::traits_type >
|
|---|
| 59 | ---
|
|---|
| 60 | > class Tr = typename std::basic_string<Char>::traits_type >
|
|---|
| 61 | 533c564
|
|---|
| 62 | < class Traits = std::basic_string<Char>::traits_type >
|
|---|
| 63 | ---
|
|---|
| 64 | > class Tr = std::basic_string<Char>::traits_type >
|
|---|
| 65 | 537a569
|
|---|
| 66 | > typedef tokenizer_detail::traits_extension<Tr> Traits;
|
|---|
| 67 | 552c584
|
|---|
| 68 | < int r = std::ispunct(E);
|
|---|
| 69 | ---
|
|---|
| 70 | > int r = Traits::ispunct(E);
|
|---|
| 71 | 564c596
|
|---|
| 72 | < int r = std::isspace(E);
|
|---|
| 73 | ---
|
|---|
| 74 | > int r = Traits::isspace(E);
|
|---|
