[Oisf-devel] Suricata 2.0dev + PF_RING 5.6.0 sporadic crashes in HTPCallbackRequest
Chris Wakelin
c.d.wakelin at reading.ac.uk
Fri Jul 19 12:37:52 UTC 2013
Hi,
I recently upgraded our Suricata instances to Suricata 2.0dev (rev
6229bfa - just a bit before the libhtp unbundling changes) and from
PF_RING 5.5.2 to 5.6.0.
We're getting sporadic crashes in both sensors; they can go for a day
without crashing, then crash three times in half an hour, so it looks
like it's triggered by some very specific traffic.
Looking in the backtrace, the ReceivePfringLoop frame suggests to me the
packet is corrupt (as far as I can tell - e.g. src and dst port are 0
and protocol 127, IPv4 addresses don't look like our local ones).
> #17 0x0000000000592c0b in ReceivePfringLoop (tv=0xc971540, data=0x7fa92cd51f00, slot=0x86ca8c0) at source-pfring.c:311
> r = 1
> packet_q_len = 4989
> ptv = 0x7fa92cd51f00
> p = 0x420d400
> hdr = {ts = {tv_sec = 1374235900, tv_usec = 231639}, caplen = 60, len = 60, extended_hdr = {timestamp_ns = 4314826383835889664, rx_direction = 1 '\001', if_index = 6, pkt_hash = 1191440499, tx = {bounce_interface = 764450176, reserved = 0x3be157bc34058000},
> parsed_header_len = 0, parsed_pkt = {dmac = "\000\000\000\000\000", smac = "\000\000\000\000\000", eth_type = 38272, vlan_id = 11664, ip_version = 166 '�', l3_proto = 127 '\177', ip_tos = 0 '\000', ip_src = {v6 = {__in6_u = {
> __u6_addr8 = "�}F\000\000\000\000\000@\025\227\f\000\000\000", __u6_addr16 = {32243, 70, 0, 0, 5440, 3223, 0, 0}, __u6_addr32 = {4619763, 0, 211227968, 0}}}, v4 = 4619763}, ip_dst = {v6 = {__in6_u = {
> __u6_addr8 = "\000\200\005\064�W�;\000\000\000\000\000\000\000", __u6_addr16 = {32768, 13317, 22460, 15329, 0, 0, 0, 0}, __u6_addr32 = {872775680, 1004623804, 0, 0}}}, v4 = 872775680}, l4_src_port = 0, l4_dst_port = 0, tcp = {flags = 0 '\000',
> seq_num = 764450208, ack_num = 32678}, tunnel = {tunnel_id = 4626108, tunneled_proto = 0 '\000', tunneled_ip_src = {v6 = {__in6_u = {__u6_addr8 = "@\025\227\f\000\000\000\000Y()Þª\177\000", __u6_addr16 = {5440, 3223, 0, 0, 10329, 56873, 32682, 0},
> __u6_addr32 = {211227968, 0, 3727239257, 32682}}}, v4 = 211227968}, tunneled_ip_dst = {v6 = {__in6_u = {__u6_addr8 = "\000\000\000\000\000\000\000\000\204\a,Þª\177\000", __u6_addr16 = {0, 0, 0, 0, 1924, 56876, 32682, 0}, __u6_addr32 = {0, 0,
> 3727427460, 32682}}}, v4 = 0}, tunneled_l4_src_port = 0, tunneled_l4_dst_port = 0}, last_matched_plugin_id = 4, last_matched_rule_id = 0, offset = {eth_offset = 5440, vlan_offset = 3223, l3_offset = 0, l4_offset = 0, payload_offset = -27168}}}}
> s = 0x86ca8c0
> last_dump = 1374235900
> current_time = {tv_sec = 1374235900, tv_usec = 231263}
> __FUNCTION__ = "ReceivePfringLoop"
I've attached a backtrace from a core that was generated a few minutes
ago (Suricata was compiled with CFLAGS="-ggdb -O0").
Any ideas what traffic caused this? (My feeling is the corrupt packets,
if that's what they are, are probably PF_RING's fault, but of course
Suricata shouldn't crash even then.)
I can downgrade Suricata, but alas I'm not allowed to touch PF_RING
without going through a Change Control process (it upset the border
switch once).
Best Wishes,
Chris
--
--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+-
Christopher Wakelin, c.d.wakelin at reading.ac.uk
IT Services Centre, The University of Reading, Tel: +44 (0)118 378 2908
Whiteknights, Reading, RG6 6AF, UK Fax: +44 (0)118 975 3094
-------------- next part --------------
GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04
Copyright (C) 2012 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
For bug reporting instructions, please see:
<http://bugs.launchpad.net/gdb-linaro/>...
Reading symbols from /opt/RDGsuricata.pf560.080713/bin/suricata...done.
[New LWP 7088]
[New LWP 7094]
[New LWP 7097]
[New LWP 7091]
[New LWP 7100]
[New LWP 7103]
[New LWP 7106]
[New LWP 7112]
[New LWP 7109]
[New LWP 7082]
[New LWP 6417]
[New LWP 7117]
[New LWP 7116]
[New LWP 7115]
[New LWP 7085]
[New LWP 7079]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/opt/RDGsuricata/bin/suricata --pfring -c /etc/suricata/suricata-dnacluster.yam'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000432682 in HTPCallbackRequest (connp=0x7fa502aba580) at app-layer-htp.c:1968
#0 0x0000000000432682 in HTPCallbackRequest (connp=0x7fa502aba580) at app-layer-htp.c:1968
htud = 0x4c4d544820445444
hstate = 0x7fa50214ef80
#1 0x00007faadf7247e4 in hook_run_all (hook=0x2855b00, data=0x7fa502aba580) at hooks.c:141
r = <optimised out>
callback = <optimised out>
i = <optimised out>
#2 0x00007faadf72ba84 in htp_connp_REQ_IDLE (connp=0x7fa502aba580) at htp_request.c:735
rc = <optimised out>
#3 0x00007faadf72bc41 in htp_connp_req_data (connp=0x7fa502aba580, timestamp=<optimised out>, data=<optimised out>, len=<optimised out>) at htp_request.c:864
rc = <optimised out>
#4 0x000000000042a7f8 in HTPHandleRequestData (f=0x7fa4fe89a000, htp_state=0x7fa50214ef80, pstate=0x7fa4fe0523d8, input=0x7fa62d9079d0 "\026\003\001", input_len=198, local_data=0x0, output=0x7fa62d907830) at app-layer-htp.c:643
r = -1
ret = 1
hstate = 0x7fa50214ef80
__FUNCTION__ = "HTPHandleRequestData"
__PRETTY_FUNCTION__ = "HTPHandleRequestData"
#5 0x000000000043b59b in AppLayerDoParse (local_data=0x0, f=0x7fa4fe89a000, app_layer_state=0x7fa50214ef80, parser_state=0x7fa4fe0523d8, input=0x7fa62d9079d0 "\026\003\001", input_len=198, parser_idx=1, proto=1) at app-layer-parser.c:887
retval = 0
result = {head = 0x0, tail = 0x0, cnt = 0}
r = 0
e = 0x0
#6 0x000000000043ba36 in AppLayerParse (local_data=0x0, f=0x7fa4fe89a000, proto=1 '\001', flags=4 '\004', input=0x7fa62d9079d0 "\026\003\001", input_len=198) at app-layer-parser.c:1093
r = 0
parser_idx = 1
p = 0x8f6820
ssn = 0x7fa5039359c0
parser_state_store = 0x7fa4fe0523c0
parser_state = 0x7fa4fe0523d8
app_layer_state = 0x7fa50214ef80
#7 0x0000000000413485 in AppLayerHandleTCPData (dp_ctx=0x9067308, f=0x7fa4fe89a000, ssn=0x7fa5039359c0, data=0x7fa62d9079d0 "\026\003\001", data_len=198, flags=4 '\004') at app-layer.c:186
r = 0
#8 0x00000000005aaa7d in StreamTcpReassembleAppLayer (tv=0xc971540, ra_ctx=0x9067300, ssn=0x7fa5039359c0, stream=0x7fa503935a10, p=0x420d400) at stream-tcp-reassemble.c:2933
flags = 4 '\004'
seg_tail = 0x7fa4fe59ad50
ra_base_seq = 4165199064
data = "\026\003\001\000\206\020\000\000\202\000\200\210\023Ámk©Ô\025±´Õ\236\"\004\004¡\202Ôá¿$(ª\034iÔdø¤:\231¸ýp\r\006§\017\071Ø\000\070mlýd«\235äF\b\002k6¾ë\216|w\021dcÏ\227\204/ç#\224×i(¸:ªUI,O\027Äp\026\064½ Rq\bIôÐQWèS¾ü¿\032¶\215\206\032\025 \223Ë\005Àò\t\210{ï2`%¨TÒÄ/©\232\062hÇ\024\003\001\000\001\001\026\003\001\000\060\212éæIªæFl\r\207?_&¤±àzwôÚñ-Ñ\025\207éUÿMÖlQ\\U¨ºù\220ë\215©\024ï\017âò\214{\000\000\000\000\000\000\000\000\000\000"...
data_len = 198
payload_offset = 0
payload_len = 198
next_seq = 4165199065
seg = 0x0
__PRETTY_FUNCTION__ = "StreamTcpReassembleAppLayer"
#9 0x00000000005ab346 in StreamTcpReassembleHandleSegmentUpdateACK (tv=0xc971540, ra_ctx=0x9067300, ssn=0x7fa5039359c0, stream=0x7fa503935a10, p=0x420d400) at stream-tcp-reassemble.c:3295
r = 0
#10 0x00000000005ab4a5 in StreamTcpReassembleHandleSegment (tv=0xc971540, ra_ctx=0x9067300, ssn=0x7fa5039359c0, stream=0x7fa5039359c8, p=0x420d400, pq=0x7fa6a29918d0) at stream-tcp-reassemble.c:3369
opposing_stream = 0x7fa503935a10
#11 0x000000000059b660 in HandleEstablishedPacketToClient (tv=0xc971540, ssn=0x7fa5039359c0, p=0x420d400, stt=0x7fa6a29918c0, pq=0x7fa6a29918d0) at stream-tcp.c:2048
zerowindowprobe = 0
#12 0x000000000059c037 in StreamTcpPacketStateEstablished (tv=0xc971540, p=0x420d400, stt=0x7fa6a29918c0, ssn=0x7fa5039359c0, pq=0x7fa6a29918d0) at stream-tcp.c:2294
No locals.
#13 0x00000000005a1db9 in StreamTcpPacket (tv=0xc971540, p=0x420d400, stt=0x7fa6a29918c0, pq=0x86ca680) at stream-tcp.c:4204
ssn = 0x7fa5039359c0
#14 0x00000000005a25d0 in StreamTcp (tv=0xc971540, p=0x420d400, data=0x7fa6a29918c0, pq=0x86ca680, postpq=0x0) at stream-tcp.c:4445
stt = 0x7fa6a29918c0
ret = TM_ECODE_OK
#15 0x00000000005bcafd in TmThreadsSlotVarRun (tv=0xc971540, p=0x420d400, slot=0x86ca780) at tm-threads.c:542
SlotFunc = 0x5a24c7 <StreamTcp>
r = TM_ECODE_OK
s = 0x86ca640
extra_p = 0x7fa62d909500
#16 0x000000000059259d in TmThreadsSlotProcessPkt (tv=0xc971540, s=0x86ca780, p=0x420d400) at tm-threads.h:139
r = TM_ECODE_OK
#17 0x0000000000592c0b in ReceivePfringLoop (tv=0xc971540, data=0x7fa92cd51f00, slot=0x86ca8c0) at source-pfring.c:311
r = 1
packet_q_len = 4989
ptv = 0x7fa92cd51f00
p = 0x420d400
hdr = {ts = {tv_sec = 1374235900, tv_usec = 231639}, caplen = 60, len = 60, extended_hdr = {timestamp_ns = 4314826383835889664, rx_direction = 1 '\001', if_index = 6, pkt_hash = 1191440499, tx = {bounce_interface = 764450176, reserved = 0x3be157bc34058000},
parsed_header_len = 0, parsed_pkt = {dmac = "\000\000\000\000\000", smac = "\000\000\000\000\000", eth_type = 38272, vlan_id = 11664, ip_version = 166 '¦', l3_proto = 127 '\177', ip_tos = 0 '\000', ip_src = {v6 = {__in6_u = {
__u6_addr8 = "ó}F\000\000\000\000\000@\025\227\f\000\000\000", __u6_addr16 = {32243, 70, 0, 0, 5440, 3223, 0, 0}, __u6_addr32 = {4619763, 0, 211227968, 0}}}, v4 = 4619763}, ip_dst = {v6 = {__in6_u = {
__u6_addr8 = "\000\200\005\064¼Wá;\000\000\000\000\000\000\000", __u6_addr16 = {32768, 13317, 22460, 15329, 0, 0, 0, 0}, __u6_addr32 = {872775680, 1004623804, 0, 0}}}, v4 = 872775680}, l4_src_port = 0, l4_dst_port = 0, tcp = {flags = 0 '\000',
seq_num = 764450208, ack_num = 32678}, tunnel = {tunnel_id = 4626108, tunneled_proto = 0 '\000', tunneled_ip_src = {v6 = {__in6_u = {__u6_addr8 = "@\025\227\f\000\000\000\000Y()Þª\177\000", __u6_addr16 = {5440, 3223, 0, 0, 10329, 56873, 32682, 0},
__u6_addr32 = {211227968, 0, 3727239257, 32682}}}, v4 = 211227968}, tunneled_ip_dst = {v6 = {__in6_u = {__u6_addr8 = "\000\000\000\000\000\000\000\000\204\a,Þª\177\000", __u6_addr16 = {0, 0, 0, 0, 1924, 56876, 32682, 0}, __u6_addr32 = {0, 0,
3727427460, 32682}}}, v4 = 0}, tunneled_l4_src_port = 0, tunneled_l4_dst_port = 0}, last_matched_plugin_id = 4, last_matched_rule_id = 0, offset = {eth_offset = 5440, vlan_offset = 3223, l3_offset = 0, l4_offset = 0, payload_offset = -27168}}}}
s = 0x86ca8c0
last_dump = 1374235900
current_time = {tv_sec = 1374235900, tv_usec = 231263}
__FUNCTION__ = "ReceivePfringLoop"
#18 0x00000000005bd434 in TmThreadsSlotPktAcqLoop (td=0xc971540) at tm-threads.c:682
tv = 0xc971540
s = 0x86ca8c0
run = 1 '\001'
r = TM_ECODE_OK
slot = 0x0
__FUNCTION__ = "TmThreadsSlotPktAcqLoop"
#19 0x00007faadea14e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
No symbol table info available.
#20 0x00007faade2c6ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#21 0x0000000000000000 in ?? ()
No symbol table info available.
More information about the Oisf-devel
mailing list