[Oisf-users] Help with 99% CPU usage

Duarte Silva duarte.silva at serializing.me
Thu May 16 09:01:26 UTC 2013


On Wednesday 15 May 2013 19:54:21 Anoop Saldanha wrote:
> On Wed, May 15, 2013 at 3:55 PM, Duarte Silva
> 
> <duarte.silva at serializing.me> wrote:
> > Hi all,
> > 
> > I'm currently facing a problem with Suricata. After running for a while,
> > there is always an AF_PACKET thread (workers mode) that hogs the CPU to
> > which it is bound to creating an awful amount of packet loss. I have
> > discarded the> 
> > following factors:
> >  - Number of rules, it has also happened without rules;
> >  - Amount of network traffic, I have seen Suricata handle ~18 MBs (150
> >  MBps) of> 
> > traffic without problems with the current configuration and it as also
> > happened with only ~2 MBs of traffic;
> > 
> >  - Memory, Suricata was only using ~500 MB of it when the CPU usage pegged
> >  to> 
> > 100%;
> > 
> > This happens repeatedly and after it happens, Suricata takes a long time
> > to
> > stop. Could some tell me what I can do to debug this issue?
> > 
> > Suricata is executed with the following command line:
> > 
> > suricata -D -c /etc/suricata/suricata.yaml --pidfile
> > /var/lock/subsys/suricata --af-packet=eth1 --user=suri --group=suri
> > 
> > I have also attached any files that can help out in debugging.
> 
> While this thread hogs the cpu, can you attach gdb to the suricata
> process, and get a bt for the specified thread, and also all the
> threads.

Follows in the attachments the traces for the hogging thread (I had to wait 
almost height hours for it to happen). I have created three traces in different 
times while the AFPacketeth12 was hoging the CPU, all of them end up in the 
list_array_get in dslib.c.

I will investigate what is happening by looking at the code, when it happens 
again I will also take traces for the other threads.
-------------- next part --------------
#0  list_array_get (_l=<value optimized out>, idx=2841) at dslib.c:242
        l = <value optimized out>
        r = 0x0
        i = 1030
#1  0x00007f664e34b6b5 in DetectEngineRunHttpRawUriMpm (
    det_ctx=0x7f663c016780, f=0x7f6640124190, htp_state=0x7f663d9f81d0,
    flags=4 '\004') at detect-engine-hrud.c:92
        cnt = <value optimized out>
        idx = 3872
        tx = <value optimized out>
        size = 4606
#2  0x00007f664e32a3fe in DetectMpmPrefilter (th_v=0x7f66501eb2a0,
    de_ctx=0x7f664fd331b0, det_ctx=0x7f663c016780, p=0x7f664f639b10)
    at detect.c:1001
No locals.
#3  SigMatchSignatures (th_v=0x7f66501eb2a0, de_ctx=0x7f664fd331b0,
    det_ctx=0x7f663c016780, p=0x7f664f639b10) at detect.c:1360
        sms_runflags = 1 '\001'
        alert_flags = 0 '\000'
        alproto = 1
        fmatch = 0
        idx = <value optimized out>
        flags = 4 '\004'
        alstate = 0x7f663d9f81d0
        smsg = 0x0
        s = 0x0
        sm = 0x0
        alversion = 12744
        reset_de_state = <value optimized out>
        app_decoder_events = <value optimized out>
        app_decoder_events_cnt = 0
        mask = <value optimized out>
#4  0x00007f664e32a75f in Detect (tv=<value optimized out>,
    p=<value optimized out>, data=<value optimized out>,
    pq=<value optimized out>, postpq=<value optimized out>)
    at detect.c:1794
        det_ctx = <value optimized out>
        de_ctx = <value optimized out>
        r = <value optimized out>
#5  0x00007f664e3fb1b8 in TmThreadsSlotVarRun (tv=0x7f66501eb2a0,
    p=0x7f664f639b10, slot=<value optimized out>) at tm-threads.c:542
        SlotFunc = <value optimized out>
        r = <value optimized out>
        s = 0x7f6650214160
        extra_p = <value optimized out>
#6  0x00007f664e3d63a0 in TmThreadsSlotProcessPkt (
    ptv=<value optimized out>) at tm-threads.h:139
        r = TM_ECODE_OK
#7  AFPReadFromRing (ptv=<value optimized out>) at source-af-packet.c:829
        p = 0x7f664f639b10
        from = <value optimized out>
        emergency_flush = 1 '\001'
        read_pkts = 33854
        loop_start = -1
#8  0x00007f664e3d68e4 in ReceiveAFPLoop (tv=0x7f66501eb2a0,
    data=0x7f663c0008e0, slot=<value optimized out>)
    at source-af-packet.c:1030
        packet_q_len = <value optimized out>
        ptv = 0x7f663c0008e0
        fds = {fd = 9, events = 1, revents = 1}
        r = <value optimized out>
        s = <value optimized out>
        last_dump = 1368693419
        current_time = {tv_sec = 1368693419, tv_usec = 960333}
        __FUNCTION__ = "ReceiveAFPLoop"
#9  0x00007f664e3fadf6 in TmThreadsSlotPktAcqLoop (td=0x7f66501eb2a0)
    at tm-threads.c:682
        tv = 0x7f66501eb2a0
        s = 0x7f66501f63c0
        run = <value optimized out>
        r = <value optimized out>
        slot = 0x0
        __FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#10 0x00007f664c83f851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#11 0x00007f664c14290d in clone () from /lib64/libc.so.6
No symbol table info available.



#0  list_array_get (_l=<value optimized out>, idx=<value optimized out>)
    at dslib.c:243
        l = <value optimized out>
        r = 0x0
        i = 93
#1  0x00007f664e34a26b in DetectEngineHHDGetBufferForTX (tx_id=521,
    de_ctx=<value optimized out>, det_ctx=0x7f6644016780,
    f=<value optimized out>, htp_state=<value optimized out>,
    flags=8 '\b', buffer_len=0x7f664a190af4) at detect-engine-hhd.c:123
        headers_buffer = 0x0
        index = 226
        tx = <value optimized out>
        headers = <value optimized out>
        h = <value optimized out>
        headers_buffer_len = <value optimized out>
        __FUNCTION__ = "DetectEngineHHDGetBufferForTX"
#2  0x00007f664e34a9ba in DetectEngineRunHttpHeaderMpm (
    det_ctx=0x7f6644016780, f=0x7f6640035cf0, htp_state=0x7f6647fd6580,
    flags=8 '\b') at detect-engine-hhd.c:215
        buffer_len = 0
        buffer = <value optimized out>
        cnt = <value optimized out>
        idx = 521
        size = 783
#3  0x00007f664e32a478 in DetectMpmPrefilter (th_v=0x7f66501f8a70,
    de_ctx=0x7f664fd331b0, det_ctx=0x7f6644016780, p=0x7f664f979fb0)
    at detect.c:1048
No locals.
#4  SigMatchSignatures (th_v=0x7f66501f8a70, de_ctx=0x7f664fd331b0,
    det_ctx=0x7f6644016780, p=0x7f664f979fb0) at detect.c:1360
        sms_runflags = 3 '\003'
        alert_flags = 0 '\000'
        alproto = 1
        fmatch = 0
        idx = <value optimized out>
        flags = 8 '\b'
        alstate = 0x7f6647fd6580
        smsg = 0x0
        s = 0x0
        sm = 0x0
        alversion = 1098
        reset_de_state = <value optimized out>
        app_decoder_events = <value optimized out>
        app_decoder_events_cnt = 0
        mask = <value optimized out>
#5  0x00007f664e32a75f in Detect (tv=<value optimized out>,
    p=<value optimized out>, data=<value optimized out>,
    pq=<value optimized out>, postpq=<value optimized out>)
    at detect.c:1794
        det_ctx = <value optimized out>
        de_ctx = <value optimized out>
        r = <value optimized out>
#6  0x00007f664e3fb1b8 in TmThreadsSlotVarRun (tv=0x7f66501f8a70,
    p=0x7f664f979fb0, slot=<value optimized out>) at tm-threads.c:542
        SlotFunc = <value optimized out>
        r = <value optimized out>
        s = 0x7f665020d100
        extra_p = <value optimized out>
#7  0x00007f664e3d63a0 in TmThreadsSlotProcessPkt (
    ptv=<value optimized out>) at tm-threads.h:139
        r = TM_ECODE_OK
#8  AFPReadFromRing (ptv=<value optimized out>) at source-af-packet.c:829
        p = 0x7f664f979fb0
        from = <value optimized out>
        emergency_flush = 0 '\000'
        read_pkts = 20
        loop_start = -1
#9  0x00007f664e3d68e4 in ReceiveAFPLoop (tv=0x7f66501f8a70,
    data=0x7f66440008e0, slot=<value optimized out>)
    at source-af-packet.c:1030
        packet_q_len = <value optimized out>
        ptv = 0x7f66440008e0
        fds = {fd = 8, events = 1, revents = 1}
        r = <value optimized out>
        s = <value optimized out>
        last_dump = 1368693402
        current_time = {tv_sec = 1368693402, tv_usec = 258900}
        __FUNCTION__ = "ReceiveAFPLoop"
#10 0x00007f664e3fadf6 in TmThreadsSlotPktAcqLoop (td=0x7f66501f8a70)
    at tm-threads.c:682
        tv = 0x7f66501f8a70
        s = 0x7f665020cf30
        run = <value optimized out>
        r = <value optimized out>
        slot = 0x0
        __FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#11 0x00007f664c83f851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#12 0x00007f664c14290d in clone () from /lib64/libc.so.6
No symbol table info available.


#0  list_array_get (_l=<value optimized out>, idx=4411) at dslib.c:242
        l = <value optimized out>
        r = 0x0
        i = 78
#1  0x00007f664e34a26b in DetectEngineHHDGetBufferForTX (tx_id=4490,
    de_ctx=<value optimized out>, det_ctx=0x7f663c016780,
    f=<value optimized out>, htp_state=<value optimized out>,
    flags=8 '\b', buffer_len=0x7f664998faf4) at detect-engine-hhd.c:123
        headers_buffer = 0x0
        index = 969
        tx = <value optimized out>
        headers = <value optimized out>
        h = <value optimized out>
        headers_buffer_len = <value optimized out>
        __FUNCTION__ = "DetectEngineHHDGetBufferForTX"
#2  0x00007f664e34a9ba in DetectEngineRunHttpHeaderMpm (
    det_ctx=0x7f663c016780, f=0x7f6640124190, htp_state=0x7f663d9f81d0,
    flags=8 '\b') at detect-engine-hhd.c:215
        buffer_len = 0
        buffer = <value optimized out>
        cnt = <value optimized out>
        idx = 4490
        size = 4606
#3  0x00007f664e32a478 in DetectMpmPrefilter (th_v=0x7f66501eb2a0,
    de_ctx=0x7f664fd331b0, det_ctx=0x7f663c016780, p=0x7f664f686df0)
    at detect.c:1048
No locals.
#4  SigMatchSignatures (th_v=0x7f66501eb2a0, de_ctx=0x7f664fd331b0,
    det_ctx=0x7f663c016780, p=0x7f664f686df0) at detect.c:1360
        sms_runflags = 3 '\003'
        alert_flags = 0 '\000'
        alproto = 1
        fmatch = 0
        idx = <value optimized out>
        flags = 8 '\b'
        alstate = 0x7f663d9f81d0
        smsg = 0x0
        s = 0x0
        sm = 0x0
        alversion = 12744
        reset_de_state = <value optimized out>
        app_decoder_events = <value optimized out>
        app_decoder_events_cnt = 0
        mask = <value optimized out>
#5  0x00007f664e32a75f in Detect (tv=<value optimized out>,
    p=<value optimized out>, data=<value optimized out>,
    pq=<value optimized out>, postpq=<value optimized out>)
    at detect.c:1794
        det_ctx = <value optimized out>
        de_ctx = <value optimized out>
        r = <value optimized out>
#6  0x00007f664e3fb1b8 in TmThreadsSlotVarRun (tv=0x7f66501eb2a0,
    p=0x7f664f686df0, slot=<value optimized out>) at tm-threads.c:542
        SlotFunc = <value optimized out>
        r = <value optimized out>
        s = 0x7f6650214160
        extra_p = <value optimized out>
#7  0x00007f664e3d63a0 in TmThreadsSlotProcessPkt (
    ptv=<value optimized out>) at tm-threads.h:139
        r = TM_ECODE_OK
#8  AFPReadFromRing (ptv=<value optimized out>) at source-af-packet.c:829
        p = 0x7f664f686df0
        from = <value optimized out>
        emergency_flush = 1 '\001'
        read_pkts = 5085
        loop_start = -1
#9  0x00007f664e3d68e4 in ReceiveAFPLoop (tv=0x7f66501eb2a0,
    data=0x7f663c0008e0, slot=<value optimized out>)
    at source-af-packet.c:1030
        packet_q_len = <value optimized out>
        ptv = 0x7f663c0008e0
        fds = {fd = 9, events = 1, revents = 1}
        r = <value optimized out>
        s = <value optimized out>
        last_dump = 1368694162
        current_time = {tv_sec = 1368694162, tv_usec = 980918}
        __FUNCTION__ = "ReceiveAFPLoop"
#10 0x00007f664e3fadf6 in TmThreadsSlotPktAcqLoop (td=0x7f66501eb2a0)
    at tm-threads.c:682
        tv = 0x7f66501eb2a0
        s = 0x7f66501f63c0
        run = <value optimized out>
        r = <value optimized out>
        slot = 0x0
        __FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#11 0x00007f664c83f851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#12 0x00007f664c14290d in clone () from /lib64/libc.so.6
No symbol table info available.





More information about the Oisf-users mailing list