diff --git a/.env.example b/.env.example index 8e3bb5e..f26194e 100644 --- a/.env.example +++ b/.env.example @@ -11,14 +11,23 @@ TRAFFIC_DIR_DOCKER="/traffic" TICK_START="2018-06-27T13:00+02:00" # Tick length in ms TICK_LENGTH=180000 +VM_IP="10.10.3.1" + +#PCAP_OVER_IP="host.docker.internal:1337" +#For multiple PCAP_OVER_IP you can comma separate +#PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050" + +############################## +# FLAGID CONFIGS +############################## + +# enable flagid scrapping +FLAGID_SCRAPE=1 +# enable flagid scanning +FLAGID_SCAN=1 # Flag Lifetime in Ticks (-1 for no check, pls don't use outside testing) FLAG_LIFETIME=-1 # Flagid endpoint currently Testendpoint in docker compose FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json" # VM IP (inside gamenet) -VM_IP="10.10.3.1" TEAM_ID="10.10.3.1" - -#PCAP_OVER_IP="host.docker.internal:1337" -#For multiple PCAP_OVER_IP you can comma separate -#PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050" \ No newline at end of file diff --git a/docker-compose-flagid.yml b/docker-compose-flagid.yml new file mode 100644 index 0000000..2dc513a --- /dev/null +++ b/docker-compose-flagid.yml @@ -0,0 +1,104 @@ +version: "3.2" +services: + mongo: + image: mongo:5 + networks: + - internal + restart: always + ports: + - "27017:27017" + + frontend: + build: + context: frontend + dockerfile: Dockerfile-frontend + image: tulip-frontend:latest + restart: unless-stopped + ports: + - "3000:3000" + depends_on: + - mongo + - api + networks: + - internal + environment: + API_SERVER_ENDPOINT: http://api:5000/ + + api: + build: + context: services/api + dockerfile: Dockerfile-api + image: tulip-api:latest + restart: unless-stopped + depends_on: + - mongo + networks: + - internal + volumes: + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro + environment: + TULIP_MONGO: ${TULIP_MONGO} + TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} + FLAG_REGEX: ${FLAG_REGEX} + TICK_START: ${TICK_START} + TICK_LENGTH: ${TICK_LENGTH} + VM_IP: ${VM_IP} + + flagids: + restart: unless-stopped + build: + context: services/flagids + image: tulip-flagids:latest + depends_on: + - mongo + networks: + - internal + environment: + TULIP_MONGO: ${TULIP_MONGO} + TICK_START: ${TICK_START} + TICK_LENGTH: ${TICK_LENGTH} + TEAM_ID: ${TEAM_ID} + FLAGID_ENDPOINT: ${FLAGID_ENDPOINT} + + assembler: + build: + context: services/go-importer + dockerfile: Dockerfile-assembler + image: tulip-assembler:latest + restart: unless-stopped + depends_on: + - mongo + networks: + - internal + volumes: + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro + command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}" + environment: + TULIP_MONGO: ${TULIP_MONGO} + FLAG_REGEX: ${FLAG_REGEX} + TICK_LENGTH: ${TICK_LENGTH} + FLAGID_SCAN: ${FLAGID_SCAN} + FLAG_LIFETIME: ${FLAG_LIFETIME} + PCAP_OVER_IP: ${PCAP_OVER_IP} + extra_hosts: + - "host.docker.internal:host-gateway" + + + enricher: + build: + context: services/go-importer + dockerfile: Dockerfile-enricher + image: tulip-enricher:latest + restart: unless-stopped + depends_on: + - mongo + networks: + - internal + volumes: + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro + command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json" + environment: + TULIP_MONGO: ${TULIP_MONGO} + +networks: + internal: diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 429bac1..cf84cf4 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -64,6 +64,7 @@ services: image: tulip-flagids:latest depends_on: - mongo + - flagidendpoint networks: - internal environment: @@ -87,9 +88,11 @@ services: - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}" environment: + DELAY: 5 TULIP_MONGO: ${TULIP_MONGO} FLAG_REGEX: ${FLAG_REGEX} TICK_LENGTH: ${TICK_LENGTH} + FLAGID_SCAN: ${FLAGID_SCAN} FLAG_LIFETIME: ${FLAG_LIFETIME} PCAP_OVER_IP: ${PCAP_OVER_IP} extra_hosts: diff --git a/docker-compose.yml b/docker-compose.yml index 2f549f1..fa133a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,22 +44,6 @@ services: TICK_LENGTH: ${TICK_LENGTH} VM_IP: ${VM_IP} - flagids: - restart: unless-stopped - build: - context: services/flagids - image: tulip-flagids:latest - depends_on: - - mongo - networks: - - internal - environment: - TULIP_MONGO: ${TULIP_MONGO} - TICK_START: ${TICK_START} - TICK_LENGTH: ${TICK_LENGTH} - TEAM_ID: ${TEAM_ID} - FLAGID_ENDPOINT: ${FLAGID_ENDPOINT} - assembler: build: context: services/go-importer @@ -77,6 +61,7 @@ services: TULIP_MONGO: ${TULIP_MONGO} FLAG_REGEX: ${FLAG_REGEX} TICK_LENGTH: ${TICK_LENGTH} + FLAGID_SCAN: ${FLAGID_SCAN} FLAG_LIFETIME: ${FLAG_LIFETIME} PCAP_OVER_IP: ${PCAP_OVER_IP} extra_hosts: diff --git a/services/go-importer/cmd/assembler/main.go b/services/go-importer/cmd/assembler/main.go index 28c3301..5ea71d2 100644 --- a/services/go-importer/cmd/assembler/main.go +++ b/services/go-importer/cmd/assembler/main.go @@ -3,16 +3,15 @@ package main import ( "go-importer/internal/pkg/db" - "fmt" - "net" - "strconv" - "flag" + "fmt" "io/ioutil" "log" + "net" "os" "os/signal" "path/filepath" + "strconv" "strings" "sync" "time" @@ -32,8 +31,6 @@ var checksum = false var nohttp = true var snaplen = 65536 -var ticklength = -1 -var flaglifetime = -1 var tstype = "" var promisc = true @@ -44,6 +41,11 @@ var pcap_over_ip = flag.String("pcap-over-ip", "", "PCAP-over-IP host + port (e. var bpf = flag.String("bpf", "", "BPF filter") var nonstrict = flag.Bool("nonstrict", false, "Do not check strict TCP / FSM flags") var experimental = flag.Bool("experimental", false, "Enable experimental features.") + +var flagid = flag.Bool("flagid", false, "Check for flagids in traffic (must be present in mong)") +var ticklength = *flag.Int("tick length", -1, "the length (in seconds) of a tick") +var flaglifetime = *flag.Int("flag lifetime", -1, "the lifetime of a flag in ticks") + var flushAfter = flag.String("flush-after", "30s", `(TCP) Connections which have buffered packets (they've gotten packets out of order and are waiting for old packets to fill the gaps) can be flushed after they're this old (their oldest gap is skipped). This is particularly useful for pcap-over-ip captures. @@ -74,14 +76,15 @@ func reassemblyCallback(entry db.FlowEntry) { ApplyFlagTags(&entry, flag_regex) } - //Apply flagid - flagids, err := g_db.GetFlagids(flaglifetime) - if err != nil { - log.Fatal(err) + //Apply flagid in / out + if *flagid { + flagids, err := g_db.GetFlagids(flaglifetime) + if err != nil { + log.Fatal(err) + } + ApplyFlagids(&entry, flagids) } - ApplyFlagids(&entry, flagids) - // Finally, insert the new entry g_db.InsertFlow(entry) } @@ -159,7 +162,7 @@ func main() { // get TICK_LENGTH strticklength := os.Getenv("TICK_LENGTH") - if strticklength != "" { + if ticklength == -1 && strticklength != "" { zwi, err := strconv.ParseInt(strticklength, 10, 64) if err != nil { log.Println("Error: ", err) @@ -170,7 +173,7 @@ func main() { // get Flag_LIFETIME strflaglifetime := os.Getenv("FLAG_LIFETIME") - if strticklength != "" { + if flaglifetime == -1 && strticklength != "" { zwi, err := strconv.Atoi(strflaglifetime) if err != nil { log.Println("Error: ", err) @@ -201,6 +204,13 @@ func main() { *pcap_over_ip = os.Getenv("PCAP_OVER_IP") } + // if flagid scans should be done + if !*flagid { + flagid_val := os.Getenv("FLAGID_SCAN") + *flagid = flagid_val != "" && flagid_val != "0" && !strings.EqualFold(flagid_val, "false") + + } + if *bpf == "" { *bpf = os.Getenv("BPF") } diff --git a/services/go-importer/cmd/assembler/tags.go b/services/go-importer/cmd/assembler/tags.go index b856cae..db384a4 100644 --- a/services/go-importer/cmd/assembler/tags.go +++ b/services/go-importer/cmd/assembler/tags.go @@ -2,9 +2,11 @@ package main import ( "go-importer/internal/pkg/db" + "log" "regexp" - "strings" + + "github.com/cloudflare/ahocorasick" ) var flagRegex *regexp.Regexp @@ -71,31 +73,41 @@ func ApplyFlagTags(flow *db.FlowEntry, reg *string) { // Apply flagids to the entire flow. // This assumes the `Data` part of the flowItem is already pre-processed, s.t. -func ApplyFlagids(flow *db.FlowEntry, flagids []db.Flagid) { +func ApplyFlagids(flow *db.FlowEntry, flagidsDb []db.Flagid) { + + var flagids []string + var matches = make(map[int]int) + + for _, flagid := range flagidsDb { + flagids = append(flagids, flagid.ID) + } + matcher := ahocorasick.NewStringMatcher(flagids) for idx := 0; idx < len(flow.Flow); idx++ { flowItem := &flow.Flow[idx] - data := flowItem.Data - for _, flagid := range flagids { - flagidstr := flagid.ID - if strings.Contains(data, flagidstr) { - var tag string - if flowItem.From == "c" { - tag = "flagid-in" - } else { - - tag = "flagid-out" - } + found := matcher.Match([]byte(flowItem.Data)) - if !contains(flow.Flagids, flagidstr) { - flow.Flagids = append(flow.Flagids, flagidstr) - } + if len(found) > 0 { + var tag string - // Add the tag if it doesn't already exist - if !contains(flow.Tags, tag) { - flow.Tags = append(flow.Tags, tag) - } + if flowItem.From == "c" { + tag = "flagid-in" + } else { + tag = "flagid-out" + } + + // Add the tag if it doesn't already exist + if !contains(flow.Tags, tag) { + flow.Tags = append(flow.Tags, tag) + } + + for _, match := range found { + matches[match] = 1 } } } + + for match, _ := range matches { + flow.Flagids = append(flow.Flagids, flagids[match]) + } } diff --git a/services/go-importer/go.mod b/services/go-importer/go.mod index e97c77b..38d358b 100644 --- a/services/go-importer/go.mod +++ b/services/go-importer/go.mod @@ -8,4 +8,5 @@ require ( github.com/google/gopacket v1.1.19 github.com/tidwall/gjson v1.14.1 go.mongodb.org/mongo-driver v1.9.1 + github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 ) diff --git a/services/go-importer/go.sum b/services/go-importer/go.sum index 499f019..dd6a23f 100644 --- a/services/go-importer/go.sum +++ b/services/go-importer/go.sum @@ -1,5 +1,7 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 h1:8yL+85JpbwrIc6m+7N1iYrjn/22z68jwrTIBOJHNe4k= +github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/services/go-importer/internal/pkg/db/db.go b/services/go-importer/internal/pkg/db/db.go index eeb1519..9aac3d5 100644 --- a/services/go-importer/internal/pkg/db/db.go +++ b/services/go-importer/internal/pkg/db/db.go @@ -76,7 +76,6 @@ func (db Database) ConfigureDatabase() { db.InsertTag("flagid-out") db.InsertTag("tcp") db.InsertTag("udp") - db.InsertTag("flagid") db.ConfigureIndexes() } @@ -183,7 +182,7 @@ func (db Database) InsertFlow(flow FlowEntry) { type PcapFile struct { FileName string `bson:"file_name"` - Position int64 `bson:"position"` + Position int64 `bson:"position"` } // Insert a new pcap uri, returns true if the pcap was not present yet, @@ -192,9 +191,9 @@ func (db Database) InsertPcap(uri string, position int64) bool { files := db.client.Database("pcap").Collection("filesImported") exists, _ := db.GetPcap(uri) if !exists { - files.InsertOne(context.TODO(), bson.M{"file_name": uri,"position": position}) + files.InsertOne(context.TODO(), bson.M{"file_name": uri, "position": position}) } else { - files.UpdateOne(context.TODO(), bson.M{"file_name": uri}, bson.M{"$set":bson.M{"position": position}}) + files.UpdateOne(context.TODO(), bson.M{"file_name": uri}, bson.M{"$set": bson.M{"position": position}}) } return !exists } diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..eb984ad --- /dev/null +++ b/start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +source .env + +if [ -n "$FLAGID_SCRAPE" ]; then + docker-compose -f docker-compose-flagid.yml up; +else + docker-compose up +fi + diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..86c10bd --- /dev/null +++ b/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker compose -f docker-compose-test.yml up --build