tcp_err_cause_hugepage_Leak

问题现象：
反复创建、删除虚机。然后巨页就越变越少。可能删除虚机时存在残留。

分析过程比较曲折，记录如下：
1，收集虚拟机已经关闭、但是巨页没有释放的时候的vmcore，进行分析：

p hstates，找到激活的巨页链表hugepage_activelist；
然后解析链表，获取页面的引用计数：
crash> list -o 0x20 -s page.flags,mapping,_count,index -x  -H   0xffffea006f000020
ffffea0066000000
  flags = 0x2fffff00004004
  mapping = 0x0
  _count = {
    counter = 0x64
  }
  index = 0x11,
ffffea00e6000000
  flags = 0x6fffff00004004
  mapping = 0x0
  _count = {
    counter = 0xe0
  }
  index = 0x11,
ffffea00d4000000
  flags = 0x6fffff00004004
  mapping = 0x0
  _count = {
    counter = 0x160
  }
  
 发现巨页的引用计数不为0，所以无法释放。但vmcore是事后的现场了，无法分析是哪里导致的巨页引用计数泄露;

2，写systemtap脚本，在申请巨页/释放巨页的地方打点，因为get_page/put_page是内核的热点函数，使用的地方非常频繁，
     打印信息也会非常频繁，几十秒钟就是几百兆日志，无法分析；
	 
3，尝试从高版本反向移植page_owner功能，依赖补丁太多，工作量很大，失败；

4，尝试参考slub的debug功能，移植代码，在get_page/put_page的时候，保持堆栈地址，开发了调试补丁；
[root@localhost SOURCES]# cat trace_hugepages.patch
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 1638e57..236cd49 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -894,6 +894,7 @@ void free_huge_page(struct page *page)
        hugepage_subpool_put_pages(spool, 1);
 }

+extern void init_page_track(struct page *s);
 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
 {
        INIT_LIST_HEAD(&page->lru);
@@ -903,6 +904,7 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
        h->nr_huge_pages++;
        h->nr_huge_pages_node[nid]++;
        spin_unlock(&hugetlb_lock);
+       init_page_track(page);
        put_page(page); /* free it into the hugepage allocator */
 }

diff --git a/mm/swap.c b/mm/swap.c
index b4d20d2..38066e6 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -31,12 +31,182 @@
 #include <linux/memcontrol.h>
 #include <linux/gfp.h>
 #include <linux/uio.h>
+#include <linux/stacktrace.h>

 #include "internal.h"

 #define CREATE_TRACE_POINTS
 #include <trace/events/pagemap.h>

+DEFINE_SPINLOCK(track_lock);
+
+#define TRACK_ADDRS_COUNT 16
+
+/*
+ * Tracking user of a slab.
+ */
+struct track {
+       unsigned long addr;     /* Called from address */
+       unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
+       int cpu;                /* Was running on cpu */
+       int pid;                /* Pid context */
+       unsigned long when;     /* When did the operation occur */
+};
+
+enum track_item { TRACK_ALLOC, TRACK_FREE };
+
+struct page_track {
+       struct page *page;
+       struct track track_alloc;
+       struct track track_free;
+};
+
+#define CGSL_MAX_TRAK 250
+static struct page_track page_tracks[CGSL_MAX_TRAK];
+static int init_flag = 0;
+
+static struct track *get_track(struct page *page,
+               enum track_item type)
+{
+       struct track *p = NULL;
+       int i;
+       u_long  flags;
+
+       if(!PageHead(page))
+               return NULL;
+
+       spin_lock_irqsave(&track_lock, flags);
+       if(!init_flag) {
+               spin_unlock_irqrestore(&track_lock,flags);
+               return NULL;
+       }
+       spin_unlock_irqrestore(&track_lock,flags);
+
+       for(i = 0; i < CGSL_MAX_TRAK; ++i){
+               spin_lock_irqsave(&track_lock, flags);
+               if(page_tracks[i].page == page){
+                       if(TRACK_ALLOC == type) {
+                               p = &page_tracks[i].track_alloc;
+                               spin_unlock_irqrestore(&track_lock, flags);
+                               return p;
+                       }
+                       else if(TRACK_FREE == type) {
+                               p = &page_tracks[i].track_free;
+                               spin_unlock_irqrestore(&track_lock, flags);
+                               return p;
+                       } else {
+                               spin_unlock_irqrestore(&track_lock, flags);
+                               return NULL;
+                       }
+               }
+               spin_unlock_irqrestore(&track_lock,flags);
+       }
+
+       return NULL;
+}
+
+static void set_track(struct page *page,
+               enum track_item alloc)
+{
+       struct track *p = get_track(page, alloc);
+       struct stack_trace trace;
+       int i;
+       u_long          flags;
+
+       if(!p)
+               return;
+       trace.nr_entries = 0;
+       trace.max_entries = TRACK_ADDRS_COUNT;
+       trace.entries = p->addrs;
+       trace.skip = 3;
+
+       spin_lock_irqsave(&track_lock, flags);
+       save_stack_trace(&trace);
+       spin_unlock_irqrestore(&track_lock, flags);
+
+       /* See rant in lockdep.c */
+       if (trace.nr_entries != 0 &&
+                       trace.entries[trace.nr_entries - 1] == ULONG_MAX)
+               trace.nr_entries--;
+
+       for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
+               p->addrs[i] = 0;
+       p->cpu = smp_processor_id();
+       p->pid = current->pid;
+       p->when = jiffies;
+}
+
+void init_page_track(struct page *s)
+{
+       int i;
+       u_long                  flags;
+
+       if(!PageHead(s))
+               return;
+
+       for(i = 0; i < CGSL_MAX_TRAK; ++i){
+               spin_lock_irqsave(&track_lock, flags);
+               if(page_tracks[i].page == NULL){
+                       page_tracks[i].page = s;
+                       memset(&page_tracks[i].track_alloc, 0, sizeof(struct track));
+                       memset(&page_tracks[i].track_free, 0, sizeof(struct track));
+                       spin_unlock_irqrestore(&track_lock, flags);
+                       return;
+               }
+               spin_unlock_irqrestore(&track_lock, flags);
+       }
+       printk(KERN_ERR "init_tracking, too many huge_pages!");
+}
+EXPORT_SYMBOL(init_page_track);
+
+void init_page_tracks( void )
+{
+       volatile  int i;
+       u_long                  flags;
+
+       spin_lock_irqsave(&track_lock, flags);
+       for(i = 0; i < CGSL_MAX_TRAK; ++i){
+               page_tracks[i].page = NULL;
+               memset(&page_tracks[i].track_alloc, 0, sizeof(struct track));
+               memset(&page_tracks[i].track_free, 0, sizeof(struct track));
+       }
+       spin_unlock_irqrestore(&track_lock, flags);
+       init_flag = 1;
+       printk(KERN_ERR "init_tracking, too many huge_pages!");
+}
+
+#if 0
+static void print_track(const char *s, struct track *t)
+{
+       printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
+               s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
+#ifdef CONFIG_STACKTRACE
+       {
+               int i;
+               for (i = 0; i < TRACK_ADDRS_COUNT; i++)
+                       if (t->addrs[i])
+                               printk(KERN_ERR "\t%pS\n", (void *)t->addrs[i]);
+                       else
+                               break;
+       }
+#endif
+}
+
+static void print_tracking(struct page *s)
+{
+       print_track("Allocated", get_track(s, TRACK_ALLOC));
+       print_track("Freed", get_track(s, TRACK_FREE));
+}
+
+static void print_page_info(struct page *page)
+{
+       printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
+               page, page->objects, page->inuse, page->freelist, page->flags);
+
+}
+#endif
+//=============================
+
 /* How many pages do we try to swap or page in/out together? */
 int page_cluster;

@@ -231,6 +401,9 @@ static void put_compound_page(struct page *page)
         *  2. THP head page.
         */
        if (likely(!PageTail(page))) {
+               if (PageHead(page))
+                       set_track(page, TRACK_FREE);
+
                if (put_page_testzero(page)) {
                        /*
                         * By the time all refcounts have been released
@@ -254,6 +427,7 @@ static void put_compound_page(struct page *page)
         *  __split_huge_page_refcount tearing down a THP page.
         */
        page_head = compound_head_by_tail(page);
+       set_track(page_head, TRACK_FREE);
        if (!__compound_tail_refcounted(page_head))
                put_unrefcounted_compound_page(page_head, page);
        else
@@ -286,6 +460,7 @@ bool __get_page_tail(struct page *page)
        unsigned long flags;
        bool got;
        struct page *page_head = compound_head(page);
+       set_track(page_head, TRACK_ALLOC);

        /* Ref to put_compound_page() comment. */
        if (!__compound_tail_refcounted(page_head)) {
@@ -1078,6 +1253,7 @@ void __init swap_setup(void)
        }
 #endif

+       init_page_tracks();
        /* Use a smaller cluster for small-memory machines */
        if (megs < 16)
                page_cluster = 2;
[root@localhost SOURCES]#


5， 把补丁部署到生产环境上，经过一周的复现，复现出了巨页泄露的问题；基于此分析：


先找到一个泄露的巨页：
ffffea00b4000000
        _count = {
          counter = 52
        }

然后根据我们的调试补丁，解析出最后一次引用这个巨页的时候的调用地址链：
    page = 0x ffffea00b4000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e472, 0xffffffff815a8d3e, 0xffffffffa05375c5, 0xffffffffa052f51a, 0xffffffffa053d763, 0xffffffffa0540080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
      cpu = 0x0,
      pid = 0x576c,
      when = 0x118b196b6
    },

然后解析addrs，查看对应的代码：
crash>
crash> kmem 0xffffffff8157e472
ffffffff8157e472 (T) tcp_sendpage+1378 /usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480

      PAGE         PHYSICAL      MAPPING       INDEX CNT FLAGS
ffffea0000055f80    157e000                0        0  1 1fffff00000400 reserved
crash> kmem 0xffffffff815a8d3e
ffffffff815a8d3e (T) inet_sendpage+110 /usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/af_inet.c: 752

crash> kmem 0xffffffffa05375c5
ffffffffa05375c5 (t) iscsi_sw_tcp_pdu_xmit+309 [iscsi_tcp]

crash>  kmem   0xffffffffa052f51a
ffffffffa052f51a (t) iscsi_tcp_task_xmit+170 [libiscsi_tcp]

crash> kmem   0xffffffffa053d763
ffffffffa053d763 (t) iscsi_xmit_task+83 [libiscsi]

crash> kmem  0xffffffffa0540080
ffffffffa0540080 (t) iscsi_xmitworker+384 [libiscsi]

crash> kmem  0xffffffff8109dedb
ffffffff8109dedb (t) process_one_work+379 /usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/kernel/workqueue.c: 2245

crash> kmem  0xffffffff810a65ff
ffffffff810a65ff (t) kthread+207 /usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/kernel/kthread.c: 200


可以看到最后是iscsi调用tcp_sendpage这个函数时，对巨页加了引用计数。

再基于此分析开源补丁，发现在tcp断链的时候，没有put_page：
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit?&id=9b42d55a66d388e4dd5550107df051a9637564fc

commit 9b42d55a66d388e4dd5550107df051a9637564fc
Author: Li RongQing <lirongqing@baidu.com>
Date:   Fri Jan 26 16:40:41 2018 +0800

    tcp: release sk_frag.page in tcp_disconnect

    socket can be disconnected and gets transformed back to a listening
    socket, if sk_frag.page is not released, which will be cloned into
    a new socket by sk_clone_lock, but the reference count of this page
    is increased, lead to a use after free or double free issue

    Signed-off-by: Li RongQing <lirongqing@baidu.com>
    Cc: Eric Dumazet <edumazet@google.com>
    Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index c8ed3a0..874c931 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2458,6 +2458,12 @@ int tcp_disconnect(struct sock *sk, int flags)

        WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);

+       if (sk->sk_frag.page) {
+               put_page(sk->sk_frag.page);
+               sk->sk_frag.page = NULL;
+               sk->sk_frag.offset = 0;
+       }
+
        sk->sk_error_report(sk);
        return err;
 }

这个补丁看上去应该可以解决这个问题
========================================================
0508 经测试，打上补丁之后，仍然有巨页泄露。
crash> p hstates
hstates = $10 =
 {{
    next_nid_to_alloc = 0,
    next_nid_to_free = 0,
    order = 18,
    mask = 18446744072635809792,
    max_huge_pages = 170,
    nr_huge_pages = 170,
    free_huge_pages = 135,
    resv_huge_pages = 0,
    surplus_huge_pages = 0,
    nr_overcommit_huge_pages = 0,
    hugepage_activelist = {
      next = 0xffffea006b000020,
      prev = 0xffffea002d000020
    },

	

crash>  list -o 0x20 -H  0xffffea006b000020
ffffea004c000000
ffffea0070000000
ffffea00d8000000
ffffea0041000000
ffffea005f000000
ffffea005a000000
ffffea0063000000
ffffea005b000000
ffffea00c2000000
ffffea00f6000000
ffffea0043000000
ffffea0062000000
ffffea004f000000
ffffea00dc000000
ffffea0048000000
ffffea0045000000
ffffea0074000000
ffffea00c1000000
ffffea00e6000000
ffffea0059000000
ffffea0044000000
ffffea004e000000
ffffea00f7000000
ffffea00f8000000
ffffea00f9000000
ffffea00fa000000
ffffea00fb000000
ffffea00fc000000
ffffea00fd000000
ffffea00fe000000
ffffea002a000000
ffffea002b000000
ffffea002c000000
ffffea002d000000
ffffffff81eaf848
crash> list -o 0x20 -H  0xffffea006b000020 | wc -l
35

34个巨页处于active状态，但实际只使用了8G个巨页，泄露了26G。

crash> list -o 0x20 -H  0xffffea006b000020 > pages.txt
crash> p page_tracks -x > dump_leak3.txt

写一个脚本来分析泄露的巨页：
#!/bin/bash
cat pages.txt | while read LINE
do
        echo  ""
        echo $LINE
	cat dump_leak3.txt  | grep -A3 $LINE
done

# ./analy.sh
执行，打印34个active 的巨页的堆栈调用情况。
ffffea004c000000
    page = 0xffffea004c000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0070000000
    page = 0xffffea0070000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00d8000000
    page = 0xffffea00d8000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0041000000
    page = 0xffffea0041000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea005f000000
    page = 0xffffea005f000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea005a000000
    page = 0xffffea005a000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0063000000
    page = 0xffffea0063000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff811b59e9, 0xffffffff81199582, 0xffffffff81193c75, 0xffffffffa09835ec, 0xffffffffa0983aae, 0xffffffffa0983f9e, 0xffffffffa04761bc, 0xffffffffa04762d3, 0xffffffffa04789fc, 0xffffffffa0999775, 0xffffffffa099de08, 0xffffffffa0984d1e, 0xffffffff811f44b5, 0xffffffff811f4731, 0xffffffff8164d909, 0x0},

ffffea005b000000
    page = 0xffffea005b000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81524b4c, 0xffffffff8158ba21, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0},

ffffea00c2000000
    page = 0xffffea00c2000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81522ba2, 0xffffffff8158a421, 0xffffffff81586ab5, 0xffffffff81587995, 0xffffffff8159240a, 0xffffffff81593bc2, 0xffffffff8156d784, 0xffffffff8156da69, 0xffffffff8156d3fd, 0xffffffff8156dd96, 0xffffffff81532252, 0xffffffff815324b8, 0xffffffff81532540, 0xffffffff815330b0, 0xffffffffa060f41d, 0xffffffffa061081a},

ffffea00f6000000
    page = 0xffffea00f6000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0043000000
    page = 0xffffea0043000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0062000000
    page = 0xffffea0062000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea004f000000
    page = 0xffffea004f000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00dc000000
    page = 0xffffea00dc000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81524b4c, 0xffffffff8158a1f2, 0xffffffff8158b6c7, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0},

ffffea0048000000
    page = 0xffffea0048000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0045000000
    page = 0xffffea0045000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0074000000
    page = 0xffffea0074000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00c1000000
    page = 0xffffea00c1000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81522ba2, 0xffffffff8158a421, 0xffffffff81586ab5, 0xffffffff815878d8, 0xffffffff8159240a, 0xffffffff81593bc2, 0xffffffff8156d784, 0xffffffff8156da69, 0xffffffff8156d3fd, 0xffffffff8156dd96, 0xffffffff81532252, 0xffffffff815324b8, 0xffffffff81532540, 0xffffffff815330b0, 0xffffffffa060f41d, 0xffffffffa061081a},

ffffea00e6000000
    page = 0xffffea00e6000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81522ba2, 0xffffffff8158a421, 0xffffffff81586ab5, 0xffffffff815878d8, 0xffffffff8159240a, 0xffffffff81593bc2, 0xffffffff8156d784, 0xffffffff8156da69, 0xffffffff8156d3fd, 0xffffffff8156dd96, 0xffffffff81532252, 0xffffffff815324b8, 0xffffffff81532540, 0xffffffff815330b0, 0xffffffffa060f41d, 0xffffffffa061081a},

ffffea0059000000
    page = 0xffffea0059000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0044000000
    page = 0xffffea0044000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81524b4c, 0xffffffff8158ba21, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0},

ffffea004e000000
    page = 0xffffea004e000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00f7000000
    page = 0xffffea00f7000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00f8000000
    page = 0xffffea00f8000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00f9000000
    page = 0xffffea00f9000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00fa000000
    page = 0xffffea00fa000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00fb000000
    page = 0xffffea00fb000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00fc000000
    page = 0xffffea00fc000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00fd000000
    page = 0xffffea00fd000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea00fe000000
    page = 0xffffea00fe000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea002a000000
    page = 0xffffea002a000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea002b000000
    page = 0xffffea002b000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea002c000000
    page = 0xffffea002c000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea002d000000
    page = 0xffffea002d000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffffff81eaf848
[root@host-192-61-11-14 ~]#


这两个地方比较特殊：

ffffea0059000000
    page = 0xffffea0059000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff8157e492, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},

ffffea0044000000
    page = 0xffffea0044000000,
    track_alloc = {
      addr = 0x0,
      addrs = {0xffffffff81524b4c, 0xffffffff8158ba21, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0},

crash>
crash> dis -l 0xffffffff8157e492
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480
0xffffffff8157e492 <tcp_sendpage+1378>: test   %al,%al
crash> dis -l 0xffffffff815a8d5e
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/af_inet.c: 752
0xffffffff815a8d5e <inet_sendpage+110>: cltq
crash> dis -l 0xffffffffa05405c5
0xffffffffa05405c5 <iscsi_sw_tcp_pdu_xmit+309>: mov    %eax,%ecx
crash> dis -l  0xffffffffa054e51a
0xffffffffa054e51a <iscsi_tcp_task_xmit+170>:   test   %eax,%eax
crash> dis -l 0xffffffffa0562763
0xffffffffa0562763 <iscsi_xmit_task+83>:        mov    %eax,%r13d
crash>
crash> dis -l 0xffffffffa0565080
0xffffffffa0565080 <iscsi_xmitworker+384>:      test   %eax,%eax
crash> dis -l 0xffffffff8109dedb
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/kernel/workqueue.c: 2245
0xffffffff8109dedb <process_one_work+379>:      nopl   0x0(%rax,%rax,1)
crash>
crash> dis -l 0xffffffff8109eeab
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/list.h: 188
0xffffffff8109eeab <worker_thread+283>: mov    0x30(%r14),%rax
crash> dis -l 0xffffffff810a65ff
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/kernel/kthread.c: 200
0xffffffff810a65ff <kthread+207>:       movslq %eax,%rdi
crash> dis -l 0xffffffff8164d858
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/arch/x86/kernel/entry_64.S: 369
0xffffffff8164d858 <ret_from_fork+88>:  movl   $0x0,0x50(%rsp)


crash>
crash>
crash> dis -l  0xffffffff81524b4c
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480
0xffffffff81524b4c <skb_split+716>:     test   %al,%al
crash> dis -l  0xffffffff8158ba21
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 1709
0xffffffff8158ba21 <tcp_write_xmit+1489>:       mov    %r13d,%edx
crash> dis -l  0xffffffff8158c3ae
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 2217
0xffffffff8158c3ae <__tcp_push_pending_frames+46>:      test   %al,%al
crash> dis -l  0xffffffff8157ad8c
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp.c: 673
0xffffffff8157ad8c <tcp_push+236>:      pop    %rbp
crash> dis -l  0xffffffff8157e424
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp.c: 969
0xffffffff8157e424 <tcp_sendpage+1268>: mov    -0x58(%rbp),%r13d
crash> dis -l  0xffffffff815a8d5e
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/af_inet.c: 752
0xffffffff815a8d5e <inet_sendpage+110>: cltq
crash>


[root@host-192-61-11-14 debug]# cat out.txt  | grep 0xffffffff8157e492 | wc -l
23
[root@host-192-61-11-14 debug]# cat out.txt  | grep 0xffffffff81524b4c  | wc -l
3
[root@host-192-61-11-14 debug]# cat out.txt  | grep 0xffffffff81522ba2  | wc -l
3
[root@host-192-61-11-14 debug]# cat out.txt  | grep 0xffffffff811b59e9  | wc -l
1
还剩下4个就巨页( addrs = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},)是一开机就分配好了的,可以不关注。

0xffffffff81524b4c的路径：
# cat out.txt  | grep 0xffffffff81524b4c
      addrs = {0xffffffff81524b4c, 0xffffffff8158ba21, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565130, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0},
      addrs = {0xffffffff81524b4c, 0xffffffff8158a1f2, 0xffffffff8158b6c7, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0},
      addrs = {0xffffffff81524b4c, 0xffffffff8158ba21, 0xffffffff8158c3ae, 0xffffffff8157ad8c, 0xffffffff8157e424, 0xffffffff815a8d5e, 0xffffffffa05405c5, 0xffffffffa054e51a, 0xffffffffa0562763, 0xffffffffa0565080, 0xffffffff8109dedb, 0xffffffff8109eeab, 0xffffffff810a65ff, 0xffffffff8164d858, 0x0, 0x0},

0xffffffff81524b4c的路径1：
crash> dis -l 0xffffffff81524b4c
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480
0xffffffff81524b4c <skb_split+716>:     test   %al,%al
crash> dis -l 0xffffffff8158ba21
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 1709
0xffffffff8158ba21 <tcp_write_xmit+1489>:       mov    %r13d,%edx
crash> dis -l  0xffffffff8158c3ae
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 2217
0xffffffff8158c3ae <__tcp_push_pending_frames+46>:      test   %al,%al
crash> dis -l  0xffffffff8157ad8c
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp.c: 673
0xffffffff8157ad8c <tcp_push+236>:      pop    %rbp
crash> dis -l  0xffffffff8157e424
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp.c: 969
0xffffffff8157e424 <tcp_sendpage+1268>: mov    -0x58(%rbp),%r13d
crash> dis -l 0xffffffff815a8d5e
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/af_inet.c: 752
0xffffffff815a8d5e <inet_sendpage+110>: cltq
crash> dis -l 0xffffffffa05405c5
0xffffffffa05405c5 <iscsi_sw_tcp_pdu_xmit+309>: mov    %eax,%ecx
crash> dis -l 0xffffffffa054e51a
0xffffffffa054e51a <iscsi_tcp_task_xmit+170>:   test   %eax,%eax
crash> dis -l  0xffffffffa0562763
0xffffffffa0562763 <iscsi_xmit_task+83>:        mov    %eax,%r13d

0xffffffff81524b4c 的路径2：
crash> dis -l 0xffffffff81524b4c
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480
0xffffffff81524b4c <skb_split+716>:     test   %al,%al
crash> dis -l  0xffffffff8158a1f2
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 1167
0xffffffff8158a1f2 <tcp_fragment+370>:  movzbl 0x7c(%rbx),%edx
crash> dis -l  0xffffffff8158b6c7
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 2012
0xffffffff8158b6c7 <tcp_write_xmit+631>:        test   %eax,%eax
crash> dis -l  0xffffffff8158c3ae
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 2217
0xffffffff8158c3ae <__tcp_push_pending_frames+46>:      test   %al,%al
crash> dis -l  0xffffffff8157ad8c
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp.c: 673
0xffffffff8157ad8c <tcp_push+236>:      pop    %rbp
crash> dis -l  0xffffffff8157e424
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp.c: 969
0xffffffff8157e424 <tcp_sendpage+1268>: mov    -0x58(%rbp),%r13d
crash> dis -l 0xffffffff815a8d5e
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/af_inet.c: 752
0xffffffff815a8d5e <inet_sendpage+110>: cltq


0xffffffff81522ba2 的路径：
# cat out.txt  | grep 0xffffffff81522ba2
      addrs = {0xffffffff81522ba2, 0xffffffff8158a421, 0xffffffff81586ab5, 0xffffffff81587995, 0xffffffff8159240a, 0xffffffff81593bc2, 0xffffffff8156d784, 0xffffffff8156da69, 0xffffffff8156d3fd, 0xffffffff8156dd96, 0xffffffff81532252, 0xffffffff815324b8, 0xffffffff81532540, 0xffffffff815330b0, 0xffffffffa060f41d, 0xffffffffa061081a},
      addrs = {0xffffffff81522ba2, 0xffffffff8158a421, 0xffffffff81586ab5, 0xffffffff815878d8, 0xffffffff8159240a, 0xffffffff81593bc2, 0xffffffff8156d784, 0xffffffff8156da69, 0xffffffff8156d3fd, 0xffffffff8156dd96, 0xffffffff81532252, 0xffffffff815324b8, 0xffffffff81532540, 0xffffffff815330b0, 0xffffffffa060f41d, 0xffffffffa061081a},
      addrs = {0xffffffff81522ba2, 0xffffffff8158a421, 0xffffffff81586ab5, 0xffffffff815878d8, 0xffffffff8159240a, 0xffffffff81593bc2, 0xffffffff8156d784, 0xffffffff8156da69, 0xffffffff8156d3fd, 0xffffffff8156dd96, 0xffffffff81532252, 0xffffffff815324b8, 0xffffffff81532540, 0xffffffff815330b0, 0xffffffffa060f41d, 0xffffffffa061081a},

0xffffffff81522ba2 的路径1：
crash> dis -l 0xffffffff81522ba2
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480
0xffffffff81522ba2 <pskb_expand_head+498>:      test   %al,%al
crash> dis -l 0xffffffff8158a421
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 1240
0xffffffff8158a421 <tcp_trim_head+209>: test   %eax,%eax
crash> dis -l  0xffffffff81586ab5
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_input.c: 3020
0xffffffff81586ab5 <tcp_ack+2517>:      test   %eax,%eax
crash> dis -l 0xffffffff81587995
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_input.c: 5250
0xffffffff81587995 <tcp_rcv_established+469>:   test   %eax,%eax
crash> dis -l  0xffffffff8159240a
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_ipv4.c: 1478
0xffffffff8159240a <tcp_v4_do_rcv+266>: jmpq   0xffffffff81592380 <tcp_v4_do_rcv+128>
crash> dis -l  0xffffffff81593bc2
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_ipv4.c: 1702
0xffffffff81593bc2 <tcp_v4_rcv+1954>:   mov    %eax,%r12d
crash> dis -l 0xffffffff8156d784
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/ip_input.c: 217
0xffffffff8156d784 <ip_local_deliver_finish+180>:       test   %eax,%eax
crash> dis -l  0xffffffff8156da69
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/ip_input.c: 259
0xffffffff8156da69 <ip_local_deliver+89>:       mov    -0x10(%rbp),%rcx
crash> dis -l  0xffffffff8156d3fd
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/ip_input.c: 372
0xffffffff8156d3fd <ip_rcv_finish+125>: pop    %rbx
crash> dis -l  0xffffffff8156dd96
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/ip_input.c: 454
0xffffffff8156dd96 <ip_rcv+694>:        jmpq   0xffffffff8156dbb4 <ip_rcv+212>
crash> dis -l  0xffffffff81532252
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/core/dev.c: 3583
0xffffffff81532252 <__netif_receive_skb_core+1410>:     mov    %eax,%r15d
crash> dis -l  0xffffffff815324b8
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/core/dev.c: 3621
0xffffffff815324b8 <__netif_receive_skb+24>:    pop    %rbx
crash> dis -l  0xffffffff81532540
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/core/dev.c: 3663
0xffffffff81532540 <netif_receive_skb+64>:      mov    %eax,%edx

0xffffffff81522ba2 的路径2：
crash> dis -l 0xffffffff81522ba2
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/include/linux/mm.h: 480
0xffffffff81522ba2 <pskb_expand_head+498>:      test   %al,%al
crash> dis -l 0xffffffff8158a421
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_output.c: 1240
0xffffffff8158a421 <tcp_trim_head+209>: test   %eax,%eax
crash> dis -l 0xffffffff81586ab5
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_input.c: 3020
0xffffffff81586ab5 <tcp_ack+2517>:      test   %eax,%eax
crash> dis -l 0xffffffff815878d8
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_input.c: 5155
0xffffffff815878d8 <tcp_rcv_established+280>:   mov    %r13,%rdi
crash> dis -l 0xffffffff8159240a
/usr/src/debug/kernel-3.10.0-327.22.2.el7/linux-3.10.0-327.22.2.el7.test.x86_64/net/ipv4/tcp_ipv4.c: 1478
0xffffffff8159240a <tcp_v4_do_rcv+266>: jmpq   0xffffffff81592380 <tcp_v4_do_rcv+128>
crash>



根据上面的情况，推测：在0xffffffff8157e492 中增加了巨页的引用计数，但在在某个分支可能缺少减引用计数的操作。

# cat out.txt  | grep -B3  0xffffffff8157e492  | grep page
    page = 0xffffea004c000000,
    page = 0xffffea0070000000,
    page = 0xffffea00d8000000,
    page = 0xffffea0041000000,
    page = 0xffffea005f000000,
    page = 0xffffea005a000000,
    page = 0xffffea00f6000000,
    page = 0xffffea0043000000,
    page = 0xffffea0062000000,
    page = 0xffffea004f000000,
    page = 0xffffea0048000000,
    page = 0xffffea0045000000,
    page = 0xffffea0074000000,
    page = 0xffffea0059000000,
    page = 0xffffea004e000000,
    page = 0xffffea00f7000000,
    page = 0xffffea00f8000000,
    page = 0xffffea00f9000000,
    page = 0xffffea00fa000000,
    page = 0xffffea00fb000000,
    page = 0xffffea00fc000000,
    page = 0xffffea00fd000000,
    page = 0xffffea00fe000000,

发现page的lru链表上面有121个元素，这个也是一个奇怪的现象：
crash>  page.lru 0xffffea0043000000
    lru = {
      next = 0xffffea0062000020,
      prev = 0xffffea00f6000020
    }
crash> list  0xffffea0062000020  | wc -l
121
crash> page.lru 0xffffea0041000000
    lru = {
      next = 0xffffea005f000020,
      prev = 0xffffea00d8000020
    }
crash> list  0xffffea005f000020 | wc -l
121

page的lru和 skb能够关联起来？


经过一段时间分析代码，发现如下这个补丁有可能会修改这个bug：
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=254d900b801fc04aa524ff7bafe28fdd1dbf0ed6

ip分段的代码中可能会泄露了skb，而skb中非线性区又持有巨页的引用。
这个补丁主要是修复headroom test的故障，但是也顺便修复了slab泄露的问题：
slowpath 分支中，consume_skb，这样就可以释放非法的skb、从而释放巨页的引用。


抓取到的堆栈：
process_one_work->iscsi_xmitworker->iscsi_xmit_task->iscsi_tcp_task_xmit->iscsi_sw_tcp_pdu_xmit->inet_sendpage->tcp_sendpage
根据代码继续推：
tcp_sendpage->tcp_push_one->tcp_write_xmit-> tso_fragment (tcp分片，一个skb拆分为多个skb)
                                          -> tcp_transmit_skb (发送拆分好的skb)->queue_xmit->ip_output->ip_finish_output->ip_fragment
这样就进入故障函数了，但是正常情况下，tcp分片只是把一个skb分为若干个skb,不会把拆分的skb放入frag_list，正常情况就是走的slow分支，不会有问题。

登录故障环境分析：
[root@host-192-61-11-2 ~]# lsmod  | grep nf_
nf_defrag_ipv6         34768  1 openvswitch
nf_nat_masquerade_ipv4    13412  1 ipt_MASQUERADE
nf_conntrack_ipv4      14862  4
nf_defrag_ipv4         12729  2 openvswitch,nf_conntrack_ipv4
nf_nat_ipv4            14115  1 iptable_nat
nf_nat                 26146  2 nf_nat_ipv4,nf_nat_masquerade_ipv4
nf_conntrack          105745  6 openvswitch,nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_ipv4

发现故障的业务环境打开了nf_defrag_ipv4   功能。
在nf_defrag_ipv4   和 nf_defrag_ipv6   功能打开的情况下，会把拆分的skb又重新合入frag_list里面，这样就会进入fast分支，形成了补丁中描述的故障场景了。
The problem is that if netfilter ipv6 defragmentation is used, skb_cow() in ip6_forward will only see reassembled skb.  Therefore, headroom is overestimated by 8 bytes (we pulled fragment header) and we don't check the skbs in the frag_list either.  We can't do these checks in netfilter defrag since outdev isn't known yet.  Furthermore, existing tests in ip6_fragment did not consider the fragment or ipv6 header size when checking headroom of the fraglist skbs.  While at it, also fix a skb leak on memory allocation -- ip6_fragment must consume the skb.
......
However similar problem still exist in ipv4.

那么场景就对应得上了，基本可以确认这个补丁可以修复它。
另外，这个补丁不仅修复了skb泄露的问题、还修复了IPV6环境下会panic的问题。