pthread_create segfaults when program is restarted after some time

[ttomas.mikalauskas@…]

Hi,

We are using boost 1.56 version on CentOS 7.4.1708 (Core)

This bug happens from time to time when we are restating application, while simulated multiple clients send message to the server.

[Switching to Thread 0x7fffeb6ac700 (LWP 27894)]
processMessageImpl (
    data_="STRING DATA"...) at main.cpp:564
564             if (!client->connected)
(gdb) bt full
#0  processMessageImpl (
    data_="CONTENT DATA DELETED"...) at main.cpp:564
        parseSucc = <optimized out>
        dataStream = {<std::basic_istream<char, std::char_traits<char> >> = {<std::basic_ios<char, std::char_traits<char> >> = {<std::ios_base> = {_vptr.ios_base = 0x7fffeb6ac700,
                static boolalpha = std::_S_boolalpha, static dec = std::_S_dec, static fixed = std::_S_fixed, static hex = std::_S_hex, static internal = std::_S_internal, static left = std::_S_left,
                static oct = std::_S_oct, static right = std::_S_right, static scientific = std::_S_scientific, static showbase = std::_S_showbase, static showpoint = std::_S_showpoint,
                static showpos = std::_S_showpos, static skipws = std::_S_skipws, static unitbuf = std::_S_unitbuf, static uppercase = std::_S_uppercase, static adjustfield = std::_S_adjustfield,
                static basefield = std::_S_basefield, static floatfield = std::_S_floatfield, static badbit = std::_S_badbit, static eofbit = std::_S_eofbit, static failbit = std::_S_failbit,
                static goodbit = std::_S_goodbit, static app = std::_S_app, static ate = std::_S_ate, static binary = std::_S_bin, static in = std::_S_in, static out = std::_S_out,
                static trunc = std::_S_trunc, static beg = std::_S_beg, static cur = std::_S_cur, static end = std::_S_end, _M_precision = 0, _M_width = 0, _M_flags = 3825362640,
                _M_exception = (std::_S_badbit | std::_S_eofbit | std::_S_failbit | unknown: 32760), _M_streambuf_state = (unknown: 3825235264), _M_callbacks = 0x6ee9a0, _M_word_zero = {
                  _M_pword = 0x7fffeb6abb30, _M_iword = 7268672}, _M_local_word = {{_M_pword = 0x0, _M_iword = 4528892}, {_M_pword = 0x0, _M_iword = 140737018779704}, {_M_pword = 0x0,
                    _M_iword = 140737351885008}, {_M_pword = 0x6ea448, _M_iword = 140737018597424}, {_M_pword = 0x0, _M_iword = 152}, {_M_pword = 0x100007f27820002, _M_iword = 0}, {_M_pword = 0x0,
                    _M_iword = 140733193388032}, {_M_pword = 0x4517b0 <tcpServer::handle_accept(session*, boost::system::error_code const&)>, _M_iword = 0}}, _M_word_size = 7268672,
                _M_word = 0x7fffe40266d0, _M_ios_locale = {static none = 0, static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32,
                  static all = 63, _M_impl = 0xffffffff, static _S_classic = 0x7ffff667be00 <(anonymous namespace)::c_locale_impl>, static _S_global = 0x7ffff667be00 <(anonymous namespace)::c_locale_impl>,
                  static _S_categories = 0x7ffff665f900 <__gnu_cxx::category_names>, static _S_once = 2}}, _M_tie = 0x7fffe40ca4c0, _M_fill = 30 '\036', _M_fill_init = false, _M_streambuf = 0x7fffeb6abbe0,
              _M_ctype = 0x6bf770, _M_num_put = 0x49182c <void boost::function1<void, std::string&>::assign_to<void (*)(std::string)>(void (*)(std::string))::stored_vtable+12>, _M_num_get = 0x1},
            _vptr.basic_istream = 0x7fffeb6ac700, _M_gcount = 0}, _M_stringbuf = {<std::basic_streambuf<char, std::char_traits<char> >> = {_vptr.basic_streambuf = 0x0, _M_in_beg = 0x7fffe40266d0 "",
              _M_in_cur = 0x7fffe4007540 "0\321I", _M_in_end = 0x6ee9a0 "\200\027", _M_out_beg = 0x7fffeb6abb30 "\002", _M_out_cur = 0x6ee940 "p\327k", _M_out_end = 0x0, _M_buf_locale = {static none = 0,
                static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63,
                _M_impl = 0x451afc <tcpServer::handle_accept(session*, boost::system::error_code const&)+844>, static _S_classic = 0x7ffff667be00 <(anonymous namespace)::c_locale_impl>,
                static _S_global = 0x7ffff667be00 <(anonymous namespace)::c_locale_impl>, static _S_categories = 0x7ffff665f900 <__gnu_cxx::category_names>, static _S_once = 2}}, _M_mode = (unknown: 0),
            _M_string = ""}}
        tree = {m_data = <error reading variable: Cannot access memory at address 0x98>, m_children = 0x9}
        spId = <error reading variable: Cannot access memory at address 0x26f8>
        url = ""
        client = 0x0
        v1_9Flag = <optimized out>
#1  0x0000000000434c8a in operator()<void (*)(std::basic_string<char>), boost::_bi::list0> (f=@0x7fffeb6abc70: 0x42f490 <processMessageImpl(std::string)>, a=<synthetic pointer>, this=0x7fffeb6abc78)
    at /usr/local/include/boost/bind/bind.hpp:253
No locals.
#2  operator() (this=0x7fffeb6abc70) at /usr/local/include/boost/bind/bind_template.hpp:20
No locals.
#3  asio_handler_invoke<boost::_bi::bind_t<void, void (*)(std::basic_string<char>), boost::_bi::list1<boost::_bi::value<std::basic_string<char> > > > > (function=...)
    at /usr/local/include/boost/asio/handler_invoke_hook.hpp:69
No locals.
#4  invoke<boost::_bi::bind_t<void, void (*)(std::basic_string<char>), boost::_bi::list1<boost::_bi::value<std::basic_string<char> > > >, boost::_bi::bind_t<void, void (*)(std::basic_string<char>), boost::_bi::list1<boost::_bi::value<std::basic_string<char> > > > > (context=..., function=...) at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:37
No locals.
#5  boost::asio::detail::completion_handler<boost::_bi::bind_t<void, void (*)(std::string), boost::_bi::list1<boost::_bi::value<std::string> > > >::do_complete (owner=0x6bf770, base=<optimized out>)
    at /usr/local/include/boost/asio/detail/completion_handler.hpp:68
        h = <optimized out>
        p = {h = <optimized out>, v = 0x0, p = 0x0}
        handler = {f_ = 0x42f490 <processMessageImpl(std::string)>, l_ = {<boost::_bi::storage1<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >> = {a1_ = {
                t_ = "CONTENT DATA DELETED"...}}, <No data fields>}}
#6  0x000000000043ca00 in complete (bytes_transferred=<optimized out>, ec=..., owner=..., this=<optimized out>) at /usr/local/include/boost/asio/detail/task_io_service_operation.hpp:38
No locals.
#7  do_run_one (ec=..., this_thread=..., lock=..., this=0x6bf770) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:372
        task_result = <optimized out>
        on_exit = {task_io_service_ = 0x6bf770, lock_ = 0x7fffeb6abcc0, this_thread_ = 0x7fffeb6abd30}
        more_handlers = true
#8  boost::asio::detail::task_io_service::run (this=0x6bf770, ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:149
        this_thread = {<boost::asio::detail::thread_info_base> = {<boost::asio::detail::noncopyable> = {<No data fields>}, reusable_memory_ = 0x7fffe40983b0},
          private_op_queue = {<boost::asio::detail::noncopyable> = {<No data fields>}, front_ = 0x0, back_ = 0x0}, private_outstanding_work = 0}
        ctx = {<boost::asio::detail::noncopyable> = {<No data fields>}, key_ = 0x6bf770, value_ = 0x7fffeb6abd30, next_ = 0x0}
        lock = {<boost::asio::detail::noncopyable> = {<No data fields>}, mutex_ = @0x6bf7a0, locked_ = false}
        n = 70
#9  0x000000000043ffb5 in boost::asio::io_service::run (this=0x6bd770 <io_service>) at /usr/local/include/boost/asio/impl/io_service.ipp:59
        ec = {m_val = 0, m_cat = 0x7ffff7dda0d0 <boost::system::system_category()::system_category_const>}
        s = <optimized out>
#10 0x00007ffff6ed030a in thread_proxy () from /lib64/libboost_thread.so.1.56.0
No symbol table info available.
#11 0x00007ffff77bae25 in start_thread (arg=0x7fffeb6ac700) at pthread_create.c:308
        __res = <optimized out>
        pd = 0x7fffeb6ac700

// Some comments here
#0  processMessageImpl (
    data_="{\"CONTENT DATA DELETED"...) at main.cpp:564
#1  0x0000000000434c8a in operator()<void (*)(std::basic_string<char>), boost::_bi::list0> (f=@0x7fffeb6abc70: 0x42f490 <processMessageImpl(std::string)>, a=<synthetic pointer>, this=0x7fffeb6abc78)
    at /usr/local/include/boost/bind/bind.hpp:253
#2  operator() (this=0x7fffeb6abc70) at /usr/local/include/boost/bind/bind_template.hpp:20
#3  asio_handler_invoke<boost::_bi::bind_t<void, void (*)(std::basic_string<char>), boost::_bi::list1<boost::_bi::value<std::basic_string<char> > > > > (function=...)
    at /usr/local/include/boost/asio/handler_invoke_hook.hpp:69
#4  invoke<boost::_bi::bind_t<void, void (*)(std::basic_string<char>), boost::_bi::list1<boost::_bi::value<std::basic_string<char> > > >, boost::_bi::bind_t<void, void (*)(std::basic_string<char>), boost::_b
i::list1<boost::_bi::value<std::basic_string<char> > > > > (context=..., function=...) at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:37
#5  boost::asio::detail::completion_handler<boost::_bi::bind_t<void, void (*)(std::string), boost::_bi::list1<boost::_bi::value<std::string> > > >::do_complete (owner=0x6bf770, base=<optimized out>)
    at /usr/local/include/boost/asio/detail/completion_handler.hpp:68
#6  0x000000000043ca00 in complete (bytes_transferred=<optimized out>, ec=..., owner=..., this=<optimized out>) at /usr/local/include/boost/asio/detail/task_io_service_operation.hpp:38
#7  do_run_one (ec=..., this_thread=..., lock=..., this=0x6bf770) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:372
#8  boost::asio::detail::task_io_service::run (this=0x6bf770, ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:149
#9  0x000000000043ffb5 in boost::asio::io_service::run (this=0x6bd770 <io_service>) at /usr/local/include/boost/asio/impl/io_service.ipp:59
#10 0x00007ffff6ed030a in thread_proxy () from /lib64/libboost_thread.so.1.56.0
#11 0x00007ffff77bae25 in start_thread (arg=0x7fffeb6ac700) at pthread_create.c:308
#12 0x00007ffff5b9234d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

[ttomas.mikalauskas@…]

Actually just realised that the problem was that we created firstly tcp server and later client in thread pool that we map to thread id.

The tcp server uses callback function to process message and process message uses map from which we get client based on thread_id

As tcp server was getting messages and was calling callback processMessage while client was not created in threadpool