LASSERT (conn->uc_peer != NULL) on mds

[slovichon]

On 2015-03-20 09:45:03 -0400, Zhihui Zhang wrote:

On orange this morning:


(gdb) bt
# 0  0x00007ffff5d4d5c9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
# 1  0x00007ffff5d4ecd8 in __GI_abort () at abort.c:90
# 2  0x00007ffff5d46536 in __assert_fail_base (fmt=0x7ffff5e960e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x7ee59b "conn->uc_peer != ((void *)0)",
    file=file@entry=0x7ee448 "/home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/conn.c", line=line@entry=664, function=function@entry=0x7ee850 <__PRETTY_FUNCTION__.23103> "usocklnd_destroy_conn")
    at assert.c:92
# 3  0x00007ffff5d465e2 in __GI___assert_fail (assertion=0x7ee59b "conn->uc_peer != ((void *)0)", file=0x7ee448 "/home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/conn.c", line=664,
    function=0x7ee850 <__PRETTY_FUNCTION__.23103> "usocklnd_destroy_conn") at assert.c:101
# 4  0x000000000043fd58 in usocklnd_destroy_conn (conn=0xf4fd40) at /home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/conn.c:664
# 5  0x0000000000595308 in usocklnd_conn_decref (conn=0xf4fd40) at /home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/usocklnd.h:254
# 6  0x0000000000597abc in usocklnd_process_pollrequest (pr=0x0, pt_data=0xf49d38) at /home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/poll.c:444
# 7  0x0000000000595457 in usocklnd_poll_thread (arg=0xf49d38) at /home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/poll.c:105
# 8  0x0000000000672a4f in psc_usklndthr_begin (thr=0x7fffdc0008c0) at /home/zhihui/projects-orange/pfl/usklndthr.c:40
# 9  0x00000000006d57c4 in _pscthr_begin (arg=0x7fffffffd350) at /home/zhihui/projects-orange/pfl/thread.c:323
# 10 0x00007ffff60e0df3 in start_thread (arg=0x7ffff0fe2700) at pthread_create.c:308
# 11 0x00007ffff5e0e1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) 


(gdb) up
# 5  0x0000000000595308 in usocklnd_conn_decref (conn=0xf4fd40) at /home/zhihui/projects-orange/lnet-lite/ulnds/socklnd/usocklnd.h:254
254                     usocklnd_destroy_conn(conn);
(gdb) p *conn
$1 = {uc_lx = 0x0, uc_type = 1, uc_activeflag = 0, uc_flip = 0, uc_state = 5, uc_peer = 0x0, uc_peerid = {nid = 562995062530843, pid = 54321}, uc_pt_idx = 3, uc_ni = 0x0, uc_preq = 0x0, uc_peer_ip = 2159436571, 
  uc_peer_port = 1023, uc_stale_list = {next = 0xf49d70, prev = 0xf49d70}, uc_rx_state = 9, uc_rx_hello = 0x7fffcc000d30, uc_rx_iov = 0xf4fdb8, uc_rx_iova = {{iov_base = 0x3968510, iov_len = 304}, {
      iov_base = 0x7ffecc6d0cc0, iov_len = 64}, {iov_base = 0x7ffecc6d0d00, iov_len = 64}, {iov_base = 0x7ffecc6d0d40, iov_len = 64}, {iov_base = 0x7ffecc6d0d80, iov_len = 64}, {iov_base = 0x7ffecc6d0dc0, 
      iov_len = 64}, {iov_base = 0x7ffecc6d0e00, iov_len = 64}, {iov_base = 0x7ffecc6d0e40, iov_len = 64}, {iov_base = 0x7ffecc6d0e80, iov_len = 64}, {iov_base = 0x7ffecc6d0ec0, iov_len = 64}, {
      iov_base = 0x7ffecc6d0f00, iov_len = 64}, {iov_base = 0x7ffecc6d0f40, iov_len = 64}, {iov_base = 0x7ffecc6d0f80, iov_len = 64}, {iov_base = 0x7ffecc6d0fc0, iov_len = 64}, {iov_base = 0x0, 
      iov_len = 0} <repeats 242 times>}, uc_rx_niov = 1, uc_rx_nob_left = 304, uc_rx_nob_wanted = 304, uc_rx_lnetmsg = 0x7ffff42baaa0, uc_rx_deadline = 1426858821, uc_rx_flag = 1, uc_rx_msg = {ksm_type = 193, 
    ksm_csum = 0, ksm_zc_cookies = {0, 0}, ksm_u = {lnetmsg = {ksnm_hdr = {dest_nid = 562995062530844, src_nid = 562995062530843, dest_pid = 54321, src_pid = 54321, type = 1, payload_length = 304, msg = {ack = {
              dst_wmd = {wh_interface_cookie = 18446744073709551615, wh_object_cookie = 18446744073709551615}, match_bits = 710037, mlength = 0}, put = {ack_wmd = {wh_interface_cookie = 18446744073709551615, 
                wh_object_cookie = 18446744073709551615}, match_bits = 710037, hdr_data = 0, ptl_index = 20, offset = 0}, get = {return_wmd = {wh_interface_cookie = 18446744073709551615, 
                wh_object_cookie = 18446744073709551615}, match_bits = 710037, ptl_index = 0, src_offset = 0, sink_length = 20}, reply = {dst_wmd = {wh_interface_cookie = 18446744073709551615, 
                wh_object_cookie = 18446744073709551615}}, hello = {incarnation = 18446744073709551615, type = 4294967295}}}, ksnm_payload = 0xf50e3c ""}}}, uc_lentry = {plh_next = 0x0, plh_prev = 0x0, 
    plh_magic = 1311693406324658740, plh_owner = 0x0}, uc_tx_list = {next = 0xf50e60, prev = 0xf50e60}, uc_zcack_list = {next = 0xf50e70, prev = 0xf50e70}, uc_tx_deadline = 1426798809, uc_tx_flag = 0, 
  uc_sending = 0, uc_tx_hello = 0x0, uc_refcount = {counter = 0}, uc_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, 
    __size = '\000' <repeats 39 times>, __align = 0}, uc_errored = 1}
(gdb)