[Oisf-users] Help with 99% CPU usage
Duarte Silva
duarte.silva at serializing.me
Thu May 16 09:01:26 UTC 2013
On Wednesday 15 May 2013 19:54:21 Anoop Saldanha wrote:
> On Wed, May 15, 2013 at 3:55 PM, Duarte Silva
>
> <duarte.silva at serializing.me> wrote:
> > Hi all,
> >
> > I'm currently facing a problem with Suricata. After running for a while,
> > there is always an AF_PACKET thread (workers mode) that hogs the CPU to
> > which it is bound to creating an awful amount of packet loss. I have
> > discarded the>
> > following factors:
> > - Number of rules, it has also happened without rules;
> > - Amount of network traffic, I have seen Suricata handle ~18 MBs (150
> > MBps) of>
> > traffic without problems with the current configuration and it as also
> > happened with only ~2 MBs of traffic;
> >
> > - Memory, Suricata was only using ~500 MB of it when the CPU usage pegged
> > to>
> > 100%;
> >
> > This happens repeatedly and after it happens, Suricata takes a long time
> > to
> > stop. Could some tell me what I can do to debug this issue?
> >
> > Suricata is executed with the following command line:
> >
> > suricata -D -c /etc/suricata/suricata.yaml --pidfile
> > /var/lock/subsys/suricata --af-packet=eth1 --user=suri --group=suri
> >
> > I have also attached any files that can help out in debugging.
>
> While this thread hogs the cpu, can you attach gdb to the suricata
> process, and get a bt for the specified thread, and also all the
> threads.
Follows in the attachments the traces for the hogging thread (I had to wait
almost height hours for it to happen). I have created three traces in different
times while the AFPacketeth12 was hoging the CPU, all of them end up in the
list_array_get in dslib.c.
I will investigate what is happening by looking at the code, when it happens
again I will also take traces for the other threads.
-------------- next part --------------
#0 list_array_get (_l=<value optimized out>, idx=2841) at dslib.c:242
l = <value optimized out>
r = 0x0
i = 1030
#1 0x00007f664e34b6b5 in DetectEngineRunHttpRawUriMpm (
det_ctx=0x7f663c016780, f=0x7f6640124190, htp_state=0x7f663d9f81d0,
flags=4 '\004') at detect-engine-hrud.c:92
cnt = <value optimized out>
idx = 3872
tx = <value optimized out>
size = 4606
#2 0x00007f664e32a3fe in DetectMpmPrefilter (th_v=0x7f66501eb2a0,
de_ctx=0x7f664fd331b0, det_ctx=0x7f663c016780, p=0x7f664f639b10)
at detect.c:1001
No locals.
#3 SigMatchSignatures (th_v=0x7f66501eb2a0, de_ctx=0x7f664fd331b0,
det_ctx=0x7f663c016780, p=0x7f664f639b10) at detect.c:1360
sms_runflags = 1 '\001'
alert_flags = 0 '\000'
alproto = 1
fmatch = 0
idx = <value optimized out>
flags = 4 '\004'
alstate = 0x7f663d9f81d0
smsg = 0x0
s = 0x0
sm = 0x0
alversion = 12744
reset_de_state = <value optimized out>
app_decoder_events = <value optimized out>
app_decoder_events_cnt = 0
mask = <value optimized out>
#4 0x00007f664e32a75f in Detect (tv=<value optimized out>,
p=<value optimized out>, data=<value optimized out>,
pq=<value optimized out>, postpq=<value optimized out>)
at detect.c:1794
det_ctx = <value optimized out>
de_ctx = <value optimized out>
r = <value optimized out>
#5 0x00007f664e3fb1b8 in TmThreadsSlotVarRun (tv=0x7f66501eb2a0,
p=0x7f664f639b10, slot=<value optimized out>) at tm-threads.c:542
SlotFunc = <value optimized out>
r = <value optimized out>
s = 0x7f6650214160
extra_p = <value optimized out>
#6 0x00007f664e3d63a0 in TmThreadsSlotProcessPkt (
ptv=<value optimized out>) at tm-threads.h:139
r = TM_ECODE_OK
#7 AFPReadFromRing (ptv=<value optimized out>) at source-af-packet.c:829
p = 0x7f664f639b10
from = <value optimized out>
emergency_flush = 1 '\001'
read_pkts = 33854
loop_start = -1
#8 0x00007f664e3d68e4 in ReceiveAFPLoop (tv=0x7f66501eb2a0,
data=0x7f663c0008e0, slot=<value optimized out>)
at source-af-packet.c:1030
packet_q_len = <value optimized out>
ptv = 0x7f663c0008e0
fds = {fd = 9, events = 1, revents = 1}
r = <value optimized out>
s = <value optimized out>
last_dump = 1368693419
current_time = {tv_sec = 1368693419, tv_usec = 960333}
__FUNCTION__ = "ReceiveAFPLoop"
#9 0x00007f664e3fadf6 in TmThreadsSlotPktAcqLoop (td=0x7f66501eb2a0)
at tm-threads.c:682
tv = 0x7f66501eb2a0
s = 0x7f66501f63c0
run = <value optimized out>
r = <value optimized out>
slot = 0x0
__FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#10 0x00007f664c83f851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#11 0x00007f664c14290d in clone () from /lib64/libc.so.6
No symbol table info available.
#0 list_array_get (_l=<value optimized out>, idx=<value optimized out>)
at dslib.c:243
l = <value optimized out>
r = 0x0
i = 93
#1 0x00007f664e34a26b in DetectEngineHHDGetBufferForTX (tx_id=521,
de_ctx=<value optimized out>, det_ctx=0x7f6644016780,
f=<value optimized out>, htp_state=<value optimized out>,
flags=8 '\b', buffer_len=0x7f664a190af4) at detect-engine-hhd.c:123
headers_buffer = 0x0
index = 226
tx = <value optimized out>
headers = <value optimized out>
h = <value optimized out>
headers_buffer_len = <value optimized out>
__FUNCTION__ = "DetectEngineHHDGetBufferForTX"
#2 0x00007f664e34a9ba in DetectEngineRunHttpHeaderMpm (
det_ctx=0x7f6644016780, f=0x7f6640035cf0, htp_state=0x7f6647fd6580,
flags=8 '\b') at detect-engine-hhd.c:215
buffer_len = 0
buffer = <value optimized out>
cnt = <value optimized out>
idx = 521
size = 783
#3 0x00007f664e32a478 in DetectMpmPrefilter (th_v=0x7f66501f8a70,
de_ctx=0x7f664fd331b0, det_ctx=0x7f6644016780, p=0x7f664f979fb0)
at detect.c:1048
No locals.
#4 SigMatchSignatures (th_v=0x7f66501f8a70, de_ctx=0x7f664fd331b0,
det_ctx=0x7f6644016780, p=0x7f664f979fb0) at detect.c:1360
sms_runflags = 3 '\003'
alert_flags = 0 '\000'
alproto = 1
fmatch = 0
idx = <value optimized out>
flags = 8 '\b'
alstate = 0x7f6647fd6580
smsg = 0x0
s = 0x0
sm = 0x0
alversion = 1098
reset_de_state = <value optimized out>
app_decoder_events = <value optimized out>
app_decoder_events_cnt = 0
mask = <value optimized out>
#5 0x00007f664e32a75f in Detect (tv=<value optimized out>,
p=<value optimized out>, data=<value optimized out>,
pq=<value optimized out>, postpq=<value optimized out>)
at detect.c:1794
det_ctx = <value optimized out>
de_ctx = <value optimized out>
r = <value optimized out>
#6 0x00007f664e3fb1b8 in TmThreadsSlotVarRun (tv=0x7f66501f8a70,
p=0x7f664f979fb0, slot=<value optimized out>) at tm-threads.c:542
SlotFunc = <value optimized out>
r = <value optimized out>
s = 0x7f665020d100
extra_p = <value optimized out>
#7 0x00007f664e3d63a0 in TmThreadsSlotProcessPkt (
ptv=<value optimized out>) at tm-threads.h:139
r = TM_ECODE_OK
#8 AFPReadFromRing (ptv=<value optimized out>) at source-af-packet.c:829
p = 0x7f664f979fb0
from = <value optimized out>
emergency_flush = 0 '\000'
read_pkts = 20
loop_start = -1
#9 0x00007f664e3d68e4 in ReceiveAFPLoop (tv=0x7f66501f8a70,
data=0x7f66440008e0, slot=<value optimized out>)
at source-af-packet.c:1030
packet_q_len = <value optimized out>
ptv = 0x7f66440008e0
fds = {fd = 8, events = 1, revents = 1}
r = <value optimized out>
s = <value optimized out>
last_dump = 1368693402
current_time = {tv_sec = 1368693402, tv_usec = 258900}
__FUNCTION__ = "ReceiveAFPLoop"
#10 0x00007f664e3fadf6 in TmThreadsSlotPktAcqLoop (td=0x7f66501f8a70)
at tm-threads.c:682
tv = 0x7f66501f8a70
s = 0x7f665020cf30
run = <value optimized out>
r = <value optimized out>
slot = 0x0
__FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#11 0x00007f664c83f851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#12 0x00007f664c14290d in clone () from /lib64/libc.so.6
No symbol table info available.
#0 list_array_get (_l=<value optimized out>, idx=4411) at dslib.c:242
l = <value optimized out>
r = 0x0
i = 78
#1 0x00007f664e34a26b in DetectEngineHHDGetBufferForTX (tx_id=4490,
de_ctx=<value optimized out>, det_ctx=0x7f663c016780,
f=<value optimized out>, htp_state=<value optimized out>,
flags=8 '\b', buffer_len=0x7f664998faf4) at detect-engine-hhd.c:123
headers_buffer = 0x0
index = 969
tx = <value optimized out>
headers = <value optimized out>
h = <value optimized out>
headers_buffer_len = <value optimized out>
__FUNCTION__ = "DetectEngineHHDGetBufferForTX"
#2 0x00007f664e34a9ba in DetectEngineRunHttpHeaderMpm (
det_ctx=0x7f663c016780, f=0x7f6640124190, htp_state=0x7f663d9f81d0,
flags=8 '\b') at detect-engine-hhd.c:215
buffer_len = 0
buffer = <value optimized out>
cnt = <value optimized out>
idx = 4490
size = 4606
#3 0x00007f664e32a478 in DetectMpmPrefilter (th_v=0x7f66501eb2a0,
de_ctx=0x7f664fd331b0, det_ctx=0x7f663c016780, p=0x7f664f686df0)
at detect.c:1048
No locals.
#4 SigMatchSignatures (th_v=0x7f66501eb2a0, de_ctx=0x7f664fd331b0,
det_ctx=0x7f663c016780, p=0x7f664f686df0) at detect.c:1360
sms_runflags = 3 '\003'
alert_flags = 0 '\000'
alproto = 1
fmatch = 0
idx = <value optimized out>
flags = 8 '\b'
alstate = 0x7f663d9f81d0
smsg = 0x0
s = 0x0
sm = 0x0
alversion = 12744
reset_de_state = <value optimized out>
app_decoder_events = <value optimized out>
app_decoder_events_cnt = 0
mask = <value optimized out>
#5 0x00007f664e32a75f in Detect (tv=<value optimized out>,
p=<value optimized out>, data=<value optimized out>,
pq=<value optimized out>, postpq=<value optimized out>)
at detect.c:1794
det_ctx = <value optimized out>
de_ctx = <value optimized out>
r = <value optimized out>
#6 0x00007f664e3fb1b8 in TmThreadsSlotVarRun (tv=0x7f66501eb2a0,
p=0x7f664f686df0, slot=<value optimized out>) at tm-threads.c:542
SlotFunc = <value optimized out>
r = <value optimized out>
s = 0x7f6650214160
extra_p = <value optimized out>
#7 0x00007f664e3d63a0 in TmThreadsSlotProcessPkt (
ptv=<value optimized out>) at tm-threads.h:139
r = TM_ECODE_OK
#8 AFPReadFromRing (ptv=<value optimized out>) at source-af-packet.c:829
p = 0x7f664f686df0
from = <value optimized out>
emergency_flush = 1 '\001'
read_pkts = 5085
loop_start = -1
#9 0x00007f664e3d68e4 in ReceiveAFPLoop (tv=0x7f66501eb2a0,
data=0x7f663c0008e0, slot=<value optimized out>)
at source-af-packet.c:1030
packet_q_len = <value optimized out>
ptv = 0x7f663c0008e0
fds = {fd = 9, events = 1, revents = 1}
r = <value optimized out>
s = <value optimized out>
last_dump = 1368694162
current_time = {tv_sec = 1368694162, tv_usec = 980918}
__FUNCTION__ = "ReceiveAFPLoop"
#10 0x00007f664e3fadf6 in TmThreadsSlotPktAcqLoop (td=0x7f66501eb2a0)
at tm-threads.c:682
tv = 0x7f66501eb2a0
s = 0x7f66501f63c0
run = <value optimized out>
r = <value optimized out>
slot = 0x0
__FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#11 0x00007f664c83f851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#12 0x00007f664c14290d in clone () from /lib64/libc.so.6
No symbol table info available.
More information about the Oisf-users
mailing list