Ticket #5086: boost_tokenizer_tolerate_negative_char.patch

boost/token_functions.hpp

@@ -218 +218 @@
     {
 #if !defined(BOOST_NO_CWCTYPE)
       if (sizeof(char_type) == 1)
-        return std::isspace(static_cast<int>(c)) != 0;
+        return std::isspace(static_cast<unsigned char>(c)) != 0;
       else
         return std::iswspace(static_cast<std::wint_t>(c)) != 0;
 #else
-      return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
+      return static_cast< unsigned >(c) <= 255 &&
+        std::isspace(static_cast<unsigned char>(c)) != 0;
 #endif
     }
 
@@ -230 +231 @@
     {
 #if !defined(BOOST_NO_CWCTYPE)
       if (sizeof(char_type) == 1)
-        return std::ispunct(static_cast<int>(c)) != 0;
+        return std::ispunct(static_cast<unsigned char>(c)) != 0;
       else
         return std::iswpunct(static_cast<std::wint_t>(c)) != 0;
 #else
-      return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
+      return static_cast< unsigned >(c) <= 255
+        && std::ispunct(static_cast<unsigned char>(c)) != 0;
 #endif
     }
   };

libs/tokenizer/test/tolerate_negative_char.cpp

@@ +1 @@
+// (c) Copyright Kazutoshi Satoda 2010.
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org/libs/tokenizer for documenation
+
+/// Test to tolerate negative chars in input
+#include <boost/tokenizer.hpp>
+#include <string>
+
+int main()
+{
+    char const negative_char = -123;
+    char const* p = &negative_char;
+    std::string token;
+    boost::char_separator<char>()(p, &negative_char + 1, token);
+    // This is a very specific test for MSVC which can detect invalid
+    // arguments for std::isXXX() (isspace(), ispunct(), ...) functions,
+    // as assertion failure.
+}

libs/tokenizer/test/Jamfile.v2

@@ -12 +12 @@
     [ run simple_example_3.cpp ]
     [ run simple_example_4.cpp ]
     [ run simple_example_5.cpp ]
+    [ run tolerate_negative_char.cpp ]
     ;