-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathfind_who_use_huge_page
More file actions
236 lines (209 loc) · 7.69 KB
/
find_who_use_huge_page
File metadata and controls
236 lines (209 loc) · 7.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
# numastat -m
Per-node system memory usage (in MBs):
Node 0 Node 1 Total
--------------- --------------- ---------------
HugePages_Total 20480.00 20480.00 40960.00
HugePages_Free 16384.00 18432.00 34816.00
HugePages_Surp 0.00 0.00 0.00
# lsof /dev/hugepages/
# lsof /dev/dvs_hugepages/
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
ovdk-ovsd 16819 root mem-R REG 0,37 4294967296 2659996728 /dev/dvs_hugepages/rtemap_0
ovdk-ovsv 16862 root mem REG 0,37 4294967296 2659996728 /dev/dvs_hugepages/rtemap_0
问题：lsof只能看到有4G巨页被使用，还有的2G找不到被谁占用，怎么找出来呢？
解决思路：生成vmcore，从vmcore中解析。
crash> hstates
hstates = $1 =
{{
next_nid_to_alloc = 0,
next_nid_to_free = 0,
order = 18,
mask = 18446744072635809792,
hstates = $1 =
{{
next_nid_to_alloc = 0,
next_nid_to_free = 0,
order = 18,
mask = 18446744072635809792,
max_huge_pages = 40,
nr_huge_pages = 40,
free_huge_pages = 34,
resv_huge_pages = 0,
surplus_huge_pages = 0,
nr_overcommit_huge_pages = 0,
hugepage_activelist = {
next = 0xffffea0037000020,
prev = 0xffffea0010000020
},
hugepage_freelists = {{
next = 0xffffea001c000020,
prev = 0xffffea0011000020
}, {
next = 0xffffea0036000020,
prev = 0xffffea0025000020
}, {
next = 0xffffffff81e96518 <hstates+120>,
prev = 0xffffffff81e96518 <hstates+120>
}, {
next = 0xffffffff81e96528 <hstates+136>,
prev = 0xffffffff81e96528 <hstates+136>
}, {
next = 0xffffffff81e96538 <hstates+152>,
prev = 0xffffffff81e96538 <hstates+152>
}, {
next = 0xffffffff81e96548 <hstates+168>,
prev = 0xffffffff81e96548 <hstates+168>
}, {
next = 0xffffffff81e96558 <hstates+184>,
prev = 0xffffffff81e96558 <hstates+184>
}, {
next = 0xffffffff81e96568 <hstates+200>,
prev = 0xffffffff81e96568 <hstates+200>
}, {
next = 0xffffffff81e96578 <hstates+216>,
prev = 0xffffffff81e96578 <hstates+216>
}, {
crash> list 0xffffea0037000020
ffffea0037000020
ffffea0038000020
ffffea000d000020
ffffea000e000020
ffffea000f000020
ffffea0010000020
ffffffff81e964e8
crash> list -H 0xffffea0037000020 -o 32 -s page.mapping
ffffea0038000000
mapping = 0xffff880285232b81
ffffea000d000000
mapping = 0xffff880e5a26a2a0
ffffea000e000000
mapping = 0xffff880e5a26a2a0
ffffea000f000000
mapping = 0xffff880e5a26a2a0
ffffea0010000000
mapping = 0xffff880e5a26a2a0
ffffffff81e964c8
mapping = 0x0
crash> tree -t rbtree -r address_space.i_mmap -o vm_area_struct.shared -s vm_area_struct.vm_mm 0xffff880e5a26a2a0
ffff8802851761b0
vm_mm = 0xffff880e594e2bc0
ffff8803142c8288
vm_mm = 0xffff88030f5644c0
crash> tree -t rbtree -r address_space.i_mmap -o vm_area_struct.shared -s vm_area_struct.vm_mm 0xffff880e5a26a2a0
ffff8802851761b0
vm_mm = 0xffff880e594e2bc0
ffff8803142c8288
vm_mm = 0xffff88030f5644c0
crash> tree -t rbtree -r address_space.i_mmap -o vm_area_struct.shared -s vm_area_struct.vm_mm 0xffff880e5a26a2a0
ffff8802851761b0
vm_mm = 0xffff880e594e2bc0
ffff8803142c8288
vm_mm = 0xffff88030f5644c0
crash> tree -t rbtree -r address_space.i_mmap -o vm_area_struct.shared -s vm_area_struct.vm_mm 0xffff880285232b81
1ffff88028522d3
tree: invalid kernel virtual address: 1ffff880285233b type: "rb_node rb_left"
crash> struct mm_struct.owner 0xffff880e594e2bc0
owner = 0xffff880104d2dc00
crash> struct mm_struct.owner 0xffff88030f5644c0
owner = 0xffff880128009700
crash> struct task_struct.pid,comm 0xffff880104d2dc00
pid = 15855
comm = "ovdk-ovsdk\000stop"
crash> struct task_struct.pid,comm 0xffff880128009700
pid = 15899
comm = "ovdk-ovsvswitch"
=---》 找到4G巨页，被ovdk-ovsdk和ovdk-ovsvswitch 占用。
crash> kmem 0xffff880285232b81
CACHE NAME OBJSIZE ALLOCATED TOTAL SLABS SSIZE
ffff88085f878000 anon_vma 56 16413 18176 284 4k
SLAB MEMORY NODE TOTAL ALLOCATED FREE
ffffea000a148c80 ffff880285232000 0 64 30 34
FREE / [ALLOCATED]
[ffff880285232b80]
---》属于匿名内存
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
ffffea000a148c80 285232000 0 ffff880285232d80 1 2fffff00000080 slab
crash>
crash> kmem 0xffff880e5a26a2a0
CACHE NAME OBJSIZE ALLOCATED TOTAL SLABS SSIZE
ffff880e5b3e6800 hugetlbfs_inode_cache 600 1074 1643 31 32k
SLAB MEMORY NODE TOTAL ALLOCATED FREE
ffffea0039689a00 ffff880e5a268000 1 53 40 13
FREE / [ALLOCATED]
[ffff880e5a26a140]
---》 0x2fffff00000080 = 0xffff880285232b81 -1
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
ffffea0039689a80 e5a26a000 0 0 0 6fffff00008000 tail
crash> struct anon_vma
struct anon_vma {
struct anon_vma *root;
struct rw_semaphore rwsem;
atomic_t refcount;
struct rb_root rb_root;
}
SIZE: 56
crash> tree -t rbtree -r anon_vma.rb_root -o anon_vma_chain.rb -s anon_vma_chain 0xffff880285232b80
ffff880e5ad40e00
struct anon_vma_chain {
vma = 0xffff880310f8e6c0,
anon_vma = 0xffff880285232b80,
same_vma = {
next = 0xffff880310f8e738,
prev = 0xffff880310f8e738
},
rb = {
__rb_parent_color = 1,
rb_right = 0x0,
rb_left = 0x0
},
rb_subtree_last = 262143
}
crash> struct vm_area_struct.vm_mm 0xffff880310f8e6c0
vm_mm = 0xffff880e5a2c12c0
crash> struct mm_struct.owner 0xffff880e5a2c12c0
owner = 0xffff880933cd7300
crash> struct task_struct.pid,comm 0xffff880933cd7300
pid = 25588
comm = "java\000)\000\000\060\000\000\000\000\000\000"
---》按照数据结构解析，找到它属于java进程。
验证：
# cat /proc/meminfo | grep -i huge
AnonHugePages: 610304 kB
HugePages_Total: 40
HugePages_Free: 34
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 1048576 kB
# rm -rf /dev/dvs_hugepages/*
# cat /proc/meminfo | grep -i huge
AnonHugePages: 610304 kB
HugePages_Total: 40
HugePages_Free: 38
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 1048576 kB
# ps -ef | grep java
root 24480 24420 0 13:53 ? 00:00:00 bash -c cd "/home/jenkins" && java -jar slave.jar
root 24485 24480 0 13:53 ? 00:00:12 java -jar slave.jar
# kill -9 24495 24480
# ps -ef | grep java
root 7237 19984 0 18:21 pts/4 00:00:00 grep --color=auto java
# cat /proc/meminfo | grep -i huge
AnonHugePages: 514048 kB
HugePages_Total: 40
HugePages_Free: 40
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 1048576 kB
---》删除巨页文件、杀掉java进程，占用的巨页都是释放了。
进一步，在另一个环境上，观察java是怎么使用巨页的：
]# ps -ef | grep java
root 25293 25223 0 Apr13 ? 00:00:00 bash -c cd "/home/jenkins" && java -jar slave.jar
root 25298 25293 0 Apr13 ? 00:01:14 java -jar slave.jar
# cat /proc/25298/maps | grep -i huge
480000000-4c0000000 rw-p 00000000 00:0c 2795576 /anon_hugepage (deleted)
6c0000000-700000000 rw-p 00000000 00:0c 2795575 /anon_hugepage (deleted)
---》可以看到是映射到maps中的。
man mmap中有这个：
MAP_HUGETLB (since Linux 2.6.32)
Allocate the mapping using "huge pages." See the Linux kernel source file Documentation/vm/hugetlbpage.txt for further information.