By allocating each PacketQueue individually avoid the risk of false
sharing when accessed from an unrelated thread.

Also this patch aligns the tranq_q array to cache line boundary.

I used this following semantic patch to change the references to
trans_q:

  @@
  identifier n;
  expression q;
  @@
  -trans_q[q].n
  +trans_q[q]->n

  @@
  expression queue;
  @@
  -&trans_q[queue]
  +trans_q[queue]

With some manual tweaking afterwards, because it was changing too many
places.

Signed-off-by: Holger Eitzenberger <holger@eitzenberger.org>

Index: oisf/src/suricata.h
===================================================================
--- oisf.orig/src/suricata.h
+++ oisf/src/suricata.h
@@ -120,7 +120,7 @@ enum {
 /* queue's between various other threads
  * XXX move to the TmQueue structure later
  */
-PacketQueue trans_q[NUM_QUEUES];
+extern PacketQueue *trans_q[NUM_QUEUES];
 
 SCDQDataQueue data_queues[NUM_QUEUES];
 /* memset to zeros, and mutex init! */
Index: oisf/src/suricata.c
===================================================================
--- oisf.orig/src/suricata.c
+++ oisf/src/suricata.c
@@ -211,6 +211,8 @@ SC_ATOMIC_DECLARE(unsigned int, engine_s
 /* Max packets processed simultaniously. */
 #define DEFAULT_MAX_PENDING_PACKETS 1024
 
+PacketQueue *trans_q[NUM_QUEUES] __ALIGNED(CACHE_LINE_SIZE);
+
 int rule_reload = 0;
 
 /** suricata engine control flags */
@@ -325,18 +327,24 @@ uint8_t print_mem_flag = 1;
 
 void GlobalInits()
 {
-    memset(trans_q, 0, sizeof(trans_q));
+	int q;
+
     memset(data_queues, 0, sizeof(data_queues));
 
     /* Initialize the trans_q mutex */
-    int blah;
     int r = 0;
-    for(blah=0;blah<256;blah++) {
-        r |= SCMutexInit(&trans_q[blah].mutex_q, NULL);
-        r |= SCCondInit(&trans_q[blah].cond_q, NULL);
+    for (q = 0; q < NUM_QUEUES; q++) {
+		trans_q[q] = calloc(1, sizeof(PacketQueue));
+		if (trans_q[q] == NULL) {
+			r = ENOMEM;
+			break;
+		}
+
+        r |= SCMutexInit(&trans_q[q]->mutex_q, NULL);
+        r |= SCCondInit(&trans_q[q]->cond_q, NULL);
 
-        r |= SCMutexInit(&data_queues[blah].mutex_q, NULL);
-        r |= SCCondInit(&data_queues[blah].cond_q, NULL);
+        r |= SCMutexInit(&data_queues[q].mutex_q, NULL);
+        r |= SCCondInit(&data_queues[q].cond_q, NULL);
    }
 
     if (r != 0) {
Index: oisf/src/counters.c
===================================================================
--- oisf.orig/src/counters.c
+++ oisf/src/counters.c
@@ -559,7 +559,7 @@ static void *SCPerfWakeupThread(void *ar
             tv->sc_perf_pctx.perf_flag = 1;
 
             if (tv->inq != NULL) {
-                q = &trans_q[tv->inq->id];
+                q = trans_q[tv->inq->id];
                 SCCondSignal(&q->cond_q);
             }
 
Index: oisf/src/tmqh-flow.c
===================================================================
--- oisf.orig/src/tmqh-flow.c
+++ oisf/src/tmqh-flow.c
@@ -83,7 +83,7 @@ void TmqhFlowRegister(void)
 /* same as 'simple' */
 Packet *TmqhInputFlow(ThreadVars *tv)
 {
-    PacketQueue *q = &trans_q[tv->inq->id];
+    PacketQueue *q = trans_q[tv->inq->id];
 
     SCPerfSyncCountersIfSignalled(tv, 0);
 
@@ -131,7 +131,7 @@ static int StoreQueueId(TmqhFlowCtx *ctx
         }
         memset(ctx->queues + (ctx->size - 1), 0, sizeof(TmqhFlowMode));
     }
-    ctx->queues[ctx->size - 1].q = &trans_q[id];
+    ctx->queues[ctx->size - 1].q = trans_q[id];
     SC_ATOMIC_INIT(ctx->queues[ctx->size - 1].total_packets);
     SC_ATOMIC_INIT(ctx->queues[ctx->size - 1].total_flows);
 
@@ -388,13 +388,13 @@ static int TmqhOutputFlowSetupCtxTest01(
     if (fctx->queues == NULL)
         goto end;
 
-    if (fctx->queues[0].q != &trans_q[0])
+    if (fctx->queues[0].q != trans_q[0])
         goto end;
-    if (fctx->queues[1].q != &trans_q[1])
+    if (fctx->queues[1].q != trans_q[1])
         goto end;
-    if (fctx->queues[2].q != &trans_q[2])
+    if (fctx->queues[2].q != trans_q[2])
         goto end;
-    if (fctx->queues[3].q != &trans_q[3])
+    if (fctx->queues[3].q != trans_q[3])
         goto end;
 
     retval = 1;
@@ -440,7 +440,7 @@ static int TmqhOutputFlowSetupCtxTest02(
     if (fctx->queues == NULL)
         goto end;
 
-    if (fctx->queues[0].q != &trans_q[0])
+    if (fctx->queues[0].q != trans_q[0])
         goto end;
 
     retval = 1;
@@ -472,13 +472,13 @@ static int TmqhOutputFlowSetupCtxTest03(
     if (fctx->queues == NULL)
         goto end;
 
-    if (fctx->queues[0].q != &trans_q[0])
+    if (fctx->queues[0].q != trans_q[0])
         goto end;
-    if (fctx->queues[1].q != &trans_q[1])
+    if (fctx->queues[1].q != trans_q[1])
         goto end;
-    if (fctx->queues[2].q != &trans_q[2])
+    if (fctx->queues[2].q != trans_q[2])
         goto end;
-    if (fctx->queues[3].q != &trans_q[3])
+    if (fctx->queues[3].q != trans_q[3])
         goto end;
 
     retval = 1;
Index: oisf/src/cuda-packet-batcher.c
===================================================================
--- oisf.orig/src/cuda-packet-batcher.c
+++ oisf/src/cuda-packet-batcher.c
@@ -131,7 +131,7 @@ static void SCCudaPBSetQueueBufferFlag(i
                "queue_buffer flag and signal the cuda TM inq.",
                profile->batching_timeout);
     queue_buffer = 1;
-    SCCondSignal(&((&trans_q[tmq_batcher_inq->id])->cond_q));
+    SCCondSignal(&((trans_q[tmq_batcher_inq->id])->cond_q));
 
     return;
 }
Index: oisf/src/detect-engine.c
===================================================================
--- oisf.orig/src/detect-engine.c
+++ oisf/src/detect-engine.c
@@ -529,7 +529,7 @@ static void *DetectEngineLiveRuleSwap(vo
                     Packet *p = PacketGetFromAlloc();
                     if (p != NULL) {
                         p->flags |= PKT_PSEUDO_STREAM_END;
-                        PacketQueue *q = &trans_q[detect_tvs[i]->inq->id];
+                        PacketQueue *q = trans_q[detect_tvs[i]->inq->id];
                         SCMutexLock(&q->mutex_q);
 
                         PacketEnqueue(q, p);
Index: oisf/src/flow-timeout.c
===================================================================
--- oisf.orig/src/flow-timeout.c
+++ oisf/src/flow-timeout.c
@@ -465,7 +465,7 @@ int FlowForceReassemblyForFlowV2(Flow *f
         PacketEnqueue(&stream_pseudo_pkt_decode_tm_slot->slot_post_pq, p3);
     SCMutexUnlock(&stream_pseudo_pkt_decode_tm_slot->slot_post_pq.mutex_q);
     if (stream_pseudo_pkt_decode_TV->inq != NULL) {
-        SCCondSignal(&trans_q[stream_pseudo_pkt_decode_TV->inq->id].cond_q);
+        SCCondSignal(&trans_q[stream_pseudo_pkt_decode_TV->inq->id]->cond_q);
     }
 
     return 1;
@@ -677,18 +677,18 @@ void FlowForceReassembly(void)
              * packet acquire by now using TmThreadDisableReceiveThreads()*/
             if (!(strlen(tv->inq->name) == strlen("packetpool") &&
                   strcasecmp(tv->inq->name, "packetpool") == 0)) {
-                PacketQueue *q = &trans_q[tv->inq->id];
+                PacketQueue *q = trans_q[tv->inq->id];
                 while (q->len != 0) {
                     usleep(100);
                 }
                 TmThreadsSetFlag(tv, THV_PAUSE);
                 if (tv->inq->q_type == 0)
-                    SCCondSignal(&trans_q[tv->inq->id].cond_q);
+                    SCCondSignal(&trans_q[tv->inq->id]->cond_q);
                 else
                     SCCondSignal(&data_queues[tv->inq->id].cond_q);
                 while (!TmThreadsCheckFlag(tv, THV_PAUSED)) {
                     if (tv->inq->q_type == 0)
-                        SCCondSignal(&trans_q[tv->inq->id].cond_q);
+                        SCCondSignal(&trans_q[tv->inq->id]->cond_q);
                     else
                         SCCondSignal(&data_queues[tv->inq->id].cond_q);
                     usleep(100);
Index: oisf/src/tm-threads.c
===================================================================
--- oisf.orig/src/tm-threads.c
+++ oisf/src/tm-threads.c
@@ -1616,7 +1616,7 @@ void TmThreadKillThread(ThreadVars *tv)
          * packet acquire by now using TmThreadDisableReceiveThreads()*/
         if (!(strlen(tv->inq->name) == strlen("packetpool") &&
               strcasecmp(tv->inq->name, "packetpool") == 0)) {
-            PacketQueue *q = &trans_q[tv->inq->id];
+            PacketQueue *q = trans_q[tv->inq->id];
             while (q->len != 0) {
                 usleep(1000);
             }
@@ -1644,7 +1644,7 @@ void TmThreadKillThread(ThreadVars *tv)
         if (tv->inq != NULL) {
             for (i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) {
                 if (tv->inq->q_type == 0)
-                    SCCondSignal(&trans_q[tv->inq->id].cond_q);
+                    SCCondSignal(&trans_q[tv->inq->id]->cond_q);
                 else
                     SCCondSignal(&data_queues[tv->inq->id].cond_q);
             }
@@ -1721,7 +1721,7 @@ void TmThreadDisableThreadsWithTMS(uint8
                  * packet acquire by now using TmThreadDisableReceiveThreads()*/
                 if (!(strlen(tv->inq->name) == strlen("packetpool") &&
                       strcasecmp(tv->inq->name, "packetpool") == 0)) {
-                    PacketQueue *q = &trans_q[tv->inq->id];
+                    PacketQueue *q = trans_q[tv->inq->id];
                     while (q->len != 0) {
                         usleep(1000);
                     }
@@ -1736,7 +1736,7 @@ void TmThreadDisableThreadsWithTMS(uint8
                 int i;
                 for (i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++) {
                     if (tv->inq->q_type == 0)
-                        SCCondSignal(&trans_q[tv->inq->id].cond_q);
+                        SCCondSignal(&trans_q[tv->inq->id]->cond_q);
                     else
                         SCCondSignal(&data_queues[tv->inq->id].cond_q);
                 }
Index: oisf/src/tmqh-simple.c
===================================================================
--- oisf.orig/src/tmqh-simple.c
+++ oisf/src/tmqh-simple.c
@@ -44,7 +44,7 @@ void TmqhSimpleRegister (void) {
 
 Packet *TmqhInputSimple(ThreadVars *t)
 {
-    PacketQueue *q = &trans_q[t->inq->id];
+    PacketQueue *q = trans_q[t->inq->id];
 
     SCPerfSyncCountersIfSignalled(t, 0);
 
@@ -74,14 +74,14 @@ void TmqhInputSimpleShutdownHandler(Thre
     }
 
     for (i = 0; i < (tv->inq->reader_cnt + tv->inq->writer_cnt); i++)
-        SCCondSignal(&trans_q[tv->inq->id].cond_q);
+        SCCondSignal(&trans_q[tv->inq->id]->cond_q);
 }
 
 void TmqhOutputSimple(ThreadVars *t, Packet *p)
 {
     SCLogDebug("Packet %p, p->root %p, alloced %s", p, p->root, p->flags & PKT_ALLOC ? "true":"false");
 
-    PacketQueue *q = &trans_q[t->outq->id];
+    PacketQueue *q = trans_q[t->outq->id];
 
     SCMutexLock(&q->mutex_q);
     PacketEnqueue(q, p);
Index: oisf/src/tm-queues.c
===================================================================
--- oisf.orig/src/tm-queues.c
+++ oisf/src/tm-queues.c
@@ -77,9 +77,9 @@ void TmqDebugList(void) {
     uint16_t i = 0;
     for (i = 0; i < tmq_id; i++) {
         /* get a lock accessing the len */
-        SCMutexLock(&trans_q[tmqs[i].id].mutex_q);
-        printf("TmqDebugList: id %" PRIu32 ", name \'%s\', len %" PRIu32 "\n", tmqs[i].id, tmqs[i].name, trans_q[tmqs[i].id].len);
-        SCMutexUnlock(&trans_q[tmqs[i].id].mutex_q);
+        SCMutexLock(&trans_q[tmqs[i].id]->mutex_q);
+        printf("TmqDebugList: id %" PRIu32 ", name \'%s\', len %" PRIu32 "\n", tmqs[i].id, tmqs[i].name, trans_q[tmqs[i].id]->len);
+        SCMutexUnlock(&trans_q[tmqs[i].id]->mutex_q);
     }
 }
 
@@ -98,7 +98,7 @@ void TmValidateQueueState(void)
     char err = FALSE;
 
     for (i = 0; i < tmq_id; i++) {
-        SCMutexLock(&trans_q[tmqs[i].id].mutex_q);
+        SCMutexLock(&trans_q[tmqs[i].id]->mutex_q);
         if (tmqs[i].reader_cnt == 0) {
             printf("Error: Queue \"%s\" doesn't have a reader\n", tmqs[i].name);
             err = TRUE;
@@ -106,7 +106,7 @@ void TmValidateQueueState(void)
             printf("Error: Queue \"%s\" doesn't have a writer\n", tmqs[i].name);
             err = TRUE;
         }
-        SCMutexUnlock(&trans_q[tmqs[i].id].mutex_q);
+        SCMutexUnlock(&trans_q[tmqs[i].id]->mutex_q);
 
         if (err == TRUE)
             goto error;
Index: oisf/src/suricata-common.h
===================================================================
--- oisf.orig/src/suricata-common.h
+++ oisf/src/suricata-common.h
@@ -214,6 +214,10 @@
 #define xstr(s) str(s)
 #define str(s) #s
 
+#define CACHE_LINE_SIZE		64
+
+#define __ALIGNED(a)		__attribute__((__aligned__(a)))
+
 /** type for the internal signature id. Since it's used in the matching engine
  *  extensively keeping this as small as possible reduces the overall memory
  *  footprint of the engine. Set to uint32_t if the engine needs to support


