WIP: bugFix background Worker and add unit tests

[weilewei]

• update BackgroundWorker's state to terminated in its destructor to avoid deadlock
• terminate main worker thread in the destructor
• add unit tests

[weilewei]

core dump backtrace:

Thread 2 "backgroundWorke" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffff6fff700 (LWP 2211315)]
0x000055555562fca6 in std::swap<std::_Any_data> (__a=..., __b=...)
    at /usr/include/c++/10/bits/move.h:198
198	      __a = _GLIBCXX_MOVE(__b);
(gdb) info threads
  Id   Target Id         Frame 
  1    Thread 0x7ffff7845a40 (LWP 2211292) "backgroundWorke" __lll_lock_wait (
    futex=futex@entry=0x7fffffffe890, private=0) at lowlevellock.c:52
* 2    Thread 0x7ffff6fff700 (LWP 2211315) "backgroundWorke" 0x000055555562fca6 in std::swap<std::_Any_data> (__a=..., __b=...)
    at /usr/include/c++/10/bits/move.h:198
(gdb) bt
#0  0x000055555562fca6 in std::swap<std::_Any_data> (__a=..., __b=...)
    at /usr/include/c++/10/bits/move.h:198
#1  0x000055555562f35f in std::function<int (void*)>::swap(std::function<int (void*)>&) (this=0x7ffff6ffca90, __x=...) at /usr/include/c++/10/bits/std_function.h:483
#2  0x000055555562e32c in std::function<int (void*)>::operator=(std::function<int (void*)> const&) (this=0x8, __x=...) at /usr/include/c++/10/bits/std_function.h:398
#3  0x000055555562d687 in terarkdb::BackgroundJob::operator= (this=0x0)
    at /data04/weile/internal/terarkdb-byted/third-party/zenfs/fs/zbd_zenfs.h:92
#4  0x000055555562873c in terarkdb::BackgroundWorker::ProcessJobs (
    this=0x7fffffffe860)
    at /data04/weile/internal/terarkdb-byted/third-party/zenfs/fs/zbd_zenfs.cc:314
#5  0x0000555555633166 in std::__invoke_impl<void, void (terarkdb::BackgroundWorker::*)(), terarkdb::BackgroundWorker*>(std::__invoke_memfun_deref, void (terarkdb::BackgroundWorker::*&&)(), terarkdb::BackgroundWorker*&&) (
    __f=<unknown type in /data04/weile/internal/terarkdb-byted/output/backgroundWorker_test, CU 0xf8f7b, DIE 0x11e4b2>, 
    __t=<unknown type in /data04/weile/internal/terarkdb-byted/output/backgroundWorker_test, CU 0xf8f7b, DIE 0x11e4c1>) at /usr/include/c++/10/bits/invoke.h:73
#6  0x00005555556330a9 in std::__invoke<void (terarkdb::BackgroundWorker::*)(), terarkdb::BackgroundWorker*>(void (terarkdb::BackgroundWorker::*&&)(), terarkdb::BackgroundWorker*&&) (
    __fn=<unknown type in /data04/weile/internal/terarkdb-byted/output/backgroundWorker_test, CU 0xf8f7b, DIE 0x11e6f3>) at /usr/include/c++/10/bits/invoke.h:95
#7  0x0000555555633019 in std::thread::_Invoker<std::tuple<void (terarkdb::BackgroundWorker::*)(), terarkdb::BackgroundWorker*> >::_M_invoke<0ul, 1ul> (
    this=0x7ffff7495568) at /usr/include/c++/10/thread:264
#8  0x0000555555632f9e in std::thread::_Invoker<std::tuple<void (terarkdb::BackgroundWorker::*)(), terarkdb::BackgroundWorker*> >::operator() (this=0x7ffff7495568)
    at /usr/include/c++/10/thread:271
#9  0x0000555555632f42 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (terarkdb::BackgroundWorker::*)(), terarkdb::BackgroundWorker*> > >::_M_run
    (this=0x7ffff7495560) at /usr/include/c++/10/thread:215
#10 0x00007ffff7c3bed0 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#11 0x00007ffff7fa0ea7 in start_thread (arg=<optimized out>)
    at pthread_create.c:477
#12 0x00007ffff7947def in clone () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) thread 1
[Switching to thread 1 (Thread 0x7ffff7845a40 (LWP 2211292))]
#0  __lll_lock_wait (futex=futex@entry=0x7fffffffe890, private=0)
    at lowlevellock.c:52
52	lowlevellock.c: No such file or directory.
(gdb) bt
#0  __lll_lock_wait (futex=futex@entry=0x7fffffffe890, private=0)
    at lowlevellock.c:52
#1  0x00007ffff7fa3843 in __GI___pthread_mutex_lock (mutex=0x7fffffffe890)
    at ../nptl/pthread_mutex_lock.c:80
#2  0x000055555562cab9 in __gthread_mutex_lock (__mutex=0x7fffffffe890)
    at /usr/include/x86_64-linux-gnu/c++/10/bits/gthr-default.h:749
#3  0x000055555562cb3a in std::mutex::lock (this=0x7fffffffe890)
    at /usr/include/c++/10/bits/std_mutex.h:100
#4  0x000055555562f293 in std::unique_lock<std::mutex>::lock (this=0x7fffffffe830)
    at /usr/include/c++/10/bits/unique_lock.h:138
#5  0x000055555562e1b5 in std::unique_lock<std::mutex>::unique_lock (
    this=0x7fffffffe830, __m=...) at /usr/include/c++/10/bits/unique_lock.h:68
#6  0x00005555556287d8 in terarkdb::BackgroundWorker::SubmitJob(std::function<int (void*)>, void*) (this=0x7fffffffe860, fn=..., 
    arg=0x55555684ee40 <terarkdb::question[abi:cxx11]>)
    at /data04/weile/internal/terarkdb-byted/third-party/zenfs/fs/zbd_zenfs.cc:325
#7  0x00005555555d646c in terarkdb::test ()
    at /data04/weile/internal/terarkdb-byted/third-party/zenfs/test/backgroundWorker_test.cc:34
#8  0x00005555555d65e8 in main (argc=1, argv=0x7fffffffeac8)
    at /data04/weile/internal/terarkdb-byted/third-party/zenfs/test/backgroundWorker_test.cc:47
(gdb) thread 2
[Switching to thread 2 (Thread 0x7ffff6fff700 (LWP 2211315))]
#0  0x000055555562fca6 in std::swap<std::_Any_data> (__a=..., __b=...)
    at /usr/include/c++/10/bits/move.h:198
198	      __a = _GLIBCXX_MOVE(__b);
(gdb) f 4
#4  0x000055555562873c in terarkdb::BackgroundWorker::ProcessJobs (
    this=0x7fffffffe860)
    at /data04/weile/internal/terarkdb-byted/third-party/zenfs/fs/zbd_zenfs.cc:314
314	        *job_now_ = jobs_.front();
(gdb) l
309	
310	  do {
311	    if (state_ == kRunning && !jobs_.empty()) {
312	      {
313	        std::unique_lock<std::mutex> lk(job_mtx_);
314	        *job_now_ = jobs_.front();
315	        jobs_.pop_front();
316	      }
317	      (*job_now_)();
318	    } else {
(gdb) p jobs_.size()
$1 = 24
(gdb) p jobs_.front()
$2 = {
  _vptr.BackgroundJob = 0x5555568261f0 <vtable for terarkdb::BackgroundJob+16>, 
  fn_ = {<std::_Maybe_unary_or_binary_function<int, void*>> = {<std::unary_function<void*, int>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {
      static _M_max_size = 16, static _M_max_align = 8, _M_functor = {
        _M_unused = {_M_object = 0x5555555d63a5 <terarkdb::deep_thought(void*)>, 
          _M_const_object = 0x5555555d63a5 <terarkdb::deep_thought(void*)>, 
          _M_function_pointer = 0x5555555d63a5 <terarkdb::deep_thought(void*)>, 
          _M_member_pointer = &virtual table offset 93824992764836, this adjustment 140737342031600}, _M_pod_data = "\245c]UUU\000\000\360FG\367\377\177\000"}, 
      _M_manager = 0x5555555d6e54 <std::_Function_handler<int (void*), int (*)(void*)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>}, 
    _M_invoker = 0x5555555d6e17 <std::_Function_handler<int (void*), int (*)(void*)>::_M_invoke(std::_Any_data const&, void*&&)>}, 
  arg_ = 0x55555684ee40 <terarkdb::question[abi:cxx11]>}

[weilewei]

close via #39 #39