Flagid Scan and Scraper can now be enabled/disabled (use start.sh)

Flagid scanner now uses Aho-Corasick
added script for testing (test.sh)
seperated flagid configs

.env.example

@@ -11,14 +11,23 @@ TRAFFIC_DIR_DOCKER="/traffic"
 TICK_START="2018-06-27T13:00+02:00"
 # Tick length in ms
 TICK_LENGTH=180000
+VM_IP="10.10.3.1"
+
+#PCAP_OVER_IP="host.docker.internal:1337"
+#For multiple PCAP_OVER_IP you can comma separate
+#PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050"
+
+##############################
+# FLAGID CONFIGS
+##############################
+
+# enable flagid scrapping
+FLAGID_SCRAPE=1
+# enable flagid scanning
+FLAGID_SCAN=1
 # Flag Lifetime in Ticks (-1 for no check, pls don't use outside testing)
 FLAG_LIFETIME=-1
 # Flagid endpoint currently Testendpoint in docker compose
 FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json"
 # VM IP (inside gamenet)
-VM_IP="10.10.3.1"
 TEAM_ID="10.10.3.1"
-
-#PCAP_OVER_IP="host.docker.internal:1337"
-#For multiple PCAP_OVER_IP you can comma separate
-#PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050"

docker-compose-flagid.yml

@@ -0,0 +1,104 @@
+version: "3.2"
+services:
+  mongo:
+    image: mongo:5
+    networks:
+      - internal
+    restart: always
+    ports:
+      - "27017:27017"
+
+  frontend:
+    build:
+      context: frontend
+      dockerfile: Dockerfile-frontend
+    image: tulip-frontend:latest
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    depends_on:
+      - mongo
+      - api
+    networks:
+      - internal
+    environment:
+      API_SERVER_ENDPOINT: http://api:5000/
+
+  api:
+    build:
+      context: services/api
+      dockerfile: Dockerfile-api
+    image: tulip-api:latest
+    restart: unless-stopped
+    depends_on:
+      - mongo
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+    environment:
+      TULIP_MONGO: ${TULIP_MONGO}
+      TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
+      FLAG_REGEX: ${FLAG_REGEX}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      VM_IP: ${VM_IP}
+
+  flagids:
+    restart: unless-stopped
+    build:
+      context: services/flagids
+    image: tulip-flagids:latest
+    depends_on:
+      - mongo
+    networks:
+      - internal
+    environment:
+      TULIP_MONGO: ${TULIP_MONGO}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      TEAM_ID: ${TEAM_ID}
+      FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
+
+  assembler:
+    build:
+      context: services/go-importer
+      dockerfile: Dockerfile-assembler
+    image: tulip-assembler:latest
+    restart: unless-stopped
+    depends_on:
+      - mongo
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+    command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}"
+    environment:
+      TULIP_MONGO: ${TULIP_MONGO}
+      FLAG_REGEX: ${FLAG_REGEX}
+      TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCAN: ${FLAGID_SCAN}
+      FLAG_LIFETIME: ${FLAG_LIFETIME}
+      PCAP_OVER_IP: ${PCAP_OVER_IP}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+
+  enricher:
+    build:
+      context: services/go-importer
+      dockerfile: Dockerfile-enricher
+    image: tulip-enricher:latest
+    restart: unless-stopped
+    depends_on:
+      - mongo
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+    command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json"
+    environment:
+      TULIP_MONGO: ${TULIP_MONGO}
+
+networks:
+  internal:

docker-compose-test.yml

@@ -64,6 +64,7 @@ services:
     image: tulip-flagids:latest
     depends_on:
       - mongo
+      - flagidendpoint
     networks:
       - internal
     environment:
@@ -87,9 +88,11 @@ services:
       - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
     command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}"
     environment:
+      DELAY: 5
       TULIP_MONGO: ${TULIP_MONGO}
       FLAG_REGEX: ${FLAG_REGEX}
       TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCAN: ${FLAGID_SCAN}
       FLAG_LIFETIME: ${FLAG_LIFETIME}
       PCAP_OVER_IP: ${PCAP_OVER_IP}
     extra_hosts:

docker-compose.yml

@@ -44,22 +44,6 @@ services:
       TICK_LENGTH: ${TICK_LENGTH}
       VM_IP: ${VM_IP}
 
-  flagids:
-    restart: unless-stopped
-    build:
-      context: services/flagids
-    image: tulip-flagids:latest
-    depends_on:
-      - mongo
-    networks:
-      - internal
-    environment:
-      TULIP_MONGO: ${TULIP_MONGO}
-      TICK_START: ${TICK_START}
-      TICK_LENGTH: ${TICK_LENGTH}
-      TEAM_ID: ${TEAM_ID}
-      FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
-
   assembler:
     build:
       context: services/go-importer
@@ -77,6 +61,7 @@ services:
       TULIP_MONGO: ${TULIP_MONGO}
       FLAG_REGEX: ${FLAG_REGEX}
       TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCAN: ${FLAGID_SCAN}
       FLAG_LIFETIME: ${FLAG_LIFETIME}
       PCAP_OVER_IP: ${PCAP_OVER_IP}
     extra_hosts:

services/go-importer/cmd/assembler/main.go

@@ -3,16 +3,15 @@ package main
 import (
 	"go-importer/internal/pkg/db"
 
-	"fmt"
-	"net"
-	"strconv"
-
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -32,8 +31,6 @@ var checksum = false
 var nohttp = true
 
 var snaplen = 65536
-var ticklength = -1
-var flaglifetime = -1
 var tstype = ""
 var promisc = true
 
@@ -44,6 +41,11 @@ var pcap_over_ip = flag.String("pcap-over-ip", "", "PCAP-over-IP host + port (e.
 var bpf = flag.String("bpf", "", "BPF filter")
 var nonstrict = flag.Bool("nonstrict", false, "Do not check strict TCP / FSM flags")
 var experimental = flag.Bool("experimental", false, "Enable experimental features.")
+
+var flagid = flag.Bool("flagid", false, "Check for flagids in traffic (must be present in mong)")
+var ticklength = *flag.Int("tick length", -1, "the length (in seconds) of a tick")
+var flaglifetime = *flag.Int("flag lifetime", -1, "the lifetime of a flag in ticks")
+
 var flushAfter = flag.String("flush-after", "30s", `(TCP) Connections which have buffered packets (they've gotten packets out of order and
 are waiting for old packets to fill the gaps) can be flushed after they're this old
 (their oldest gap is skipped). This is particularly useful for pcap-over-ip captures.
@@ -74,14 +76,15 @@ func reassemblyCallback(entry db.FlowEntry) {
 		ApplyFlagTags(&entry, flag_regex)
 	}
 
-	//Apply flagid
-	flagids, err := g_db.GetFlagids(flaglifetime)
-	if err != nil {
-		log.Fatal(err)
+	//Apply flagid in / out
+	if *flagid {
+		flagids, err := g_db.GetFlagids(flaglifetime)
+		if err != nil {
+			log.Fatal(err)
+		}
+		ApplyFlagids(&entry, flagids)
 	}
 
-	ApplyFlagids(&entry, flagids)
-
 	// Finally, insert the new entry
 	g_db.InsertFlow(entry)
 }
@@ -159,7 +162,7 @@ func main() {
 
 	// get TICK_LENGTH
 	strticklength := os.Getenv("TICK_LENGTH")
-	if strticklength != "" {
+	if ticklength == -1 && strticklength != "" {
 		zwi, err := strconv.ParseInt(strticklength, 10, 64)
 		if err != nil {
 			log.Println("Error: ", err)
@@ -170,7 +173,7 @@ func main() {
 
 	// get Flag_LIFETIME
 	strflaglifetime := os.Getenv("FLAG_LIFETIME")
-	if strticklength != "" {
+	if flaglifetime == -1 && strticklength != "" {
 		zwi, err := strconv.Atoi(strflaglifetime)
 		if err != nil {
 			log.Println("Error: ", err)
@@ -201,6 +204,13 @@ func main() {
 		*pcap_over_ip = os.Getenv("PCAP_OVER_IP")
 	}
 
+	// if flagid scans should be done
+	if !*flagid {
+		flagid_val := os.Getenv("FLAGID_SCAN")
+		*flagid = flagid_val != "" && flagid_val != "0" && !strings.EqualFold(flagid_val, "false")
+
+	}
+
 	if *bpf == "" {
 		*bpf = os.Getenv("BPF")
 	}

services/go-importer/cmd/assembler/tags.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"go-importer/internal/pkg/db"
+
 	"log"
 	"regexp"
-	"strings"
+
+	"github.com/cloudflare/ahocorasick"
 )
 
 var flagRegex *regexp.Regexp
@@ -71,31 +73,41 @@ func ApplyFlagTags(flow *db.FlowEntry, reg *string) {
 
 // Apply flagids to the entire flow.
 // This assumes the `Data` part of the flowItem is already pre-processed, s.t.
-func ApplyFlagids(flow *db.FlowEntry, flagids []db.Flagid) {
+func ApplyFlagids(flow *db.FlowEntry, flagidsDb []db.Flagid) {
+
+	var flagids []string
+	var matches = make(map[int]int)
+
+	for _, flagid := range flagidsDb {
+		flagids = append(flagids, flagid.ID)
+	}
 
+	matcher := ahocorasick.NewStringMatcher(flagids)
 	for idx := 0; idx < len(flow.Flow); idx++ {
 		flowItem := &flow.Flow[idx]
-		data := flowItem.Data
-		for _, flagid := range flagids {
-			flagidstr := flagid.ID
-			if strings.Contains(data, flagidstr) {
-				var tag string
-				if flowItem.From == "c" {
-					tag = "flagid-in"
-				} else {
-
-					tag = "flagid-out"
-				}
+		found := matcher.Match([]byte(flowItem.Data))
 
-				if !contains(flow.Flagids, flagidstr) {
-					flow.Flagids = append(flow.Flagids, flagidstr)
-				}
+		if len(found) > 0 {
+			var tag string
 
-				// Add the tag if it doesn't already exist
-				if !contains(flow.Tags, tag) {
-					flow.Tags = append(flow.Tags, tag)
-				}
+			if flowItem.From == "c" {
+				tag = "flagid-in"
+			} else {
+				tag = "flagid-out"
+			}
+
+			// Add the tag if it doesn't already exist
+			if !contains(flow.Tags, tag) {
+				flow.Tags = append(flow.Tags, tag)
+			}
+
+			for _, match := range found {
+				matches[match] = 1
 			}
 		}
 	}
+
+	for match, _ := range matches {
+		flow.Flagids = append(flow.Flagids, flagids[match])
+	}
 }

services/go-importer/go.mod

@@ -8,4 +8,5 @@ require (
 	github.com/google/gopacket v1.1.19
 	github.com/tidwall/gjson v1.14.1
 	go.mongodb.org/mongo-driver v1.9.1
+	github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184
 )

services/go-importer/go.sum

@@ -1,5 +1,7 @@
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 h1:8yL+85JpbwrIc6m+7N1iYrjn/22z68jwrTIBOJHNe4k=
+github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

services/go-importer/internal/pkg/db/db.go

@@ -76,7 +76,6 @@ func (db Database) ConfigureDatabase() {
 	db.InsertTag("flagid-out")
 	db.InsertTag("tcp")
 	db.InsertTag("udp")
-	db.InsertTag("flagid")
 	db.ConfigureIndexes()
 }
 
@@ -183,7 +182,7 @@ func (db Database) InsertFlow(flow FlowEntry) {
 
 type PcapFile struct {
 	FileName string `bson:"file_name"`
-	Position int64 `bson:"position"`
+	Position int64  `bson:"position"`
 }
 
 // Insert a new pcap uri, returns true if the pcap was not present yet,
@@ -192,9 +191,9 @@ func (db Database) InsertPcap(uri string, position int64) bool {
 	files := db.client.Database("pcap").Collection("filesImported")
 	exists, _ := db.GetPcap(uri)
 	if !exists {
-		files.InsertOne(context.TODO(), bson.M{"file_name": uri,"position": position})
+		files.InsertOne(context.TODO(), bson.M{"file_name": uri, "position": position})
 	} else {
-		files.UpdateOne(context.TODO(), bson.M{"file_name": uri}, bson.M{"$set":bson.M{"position": position}})
+		files.UpdateOne(context.TODO(), bson.M{"file_name": uri}, bson.M{"$set": bson.M{"position": position}})
 	}
 	return !exists
 }