feat: add support for otel/kafka driver and export (#62)

CHANGELOG.md

@@ -15,6 +15,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+## [0.7.0] - 2024-04-30
+
+### Added
+
+- Kafka driver and exporter for Otel record logs
+- Policy engine backend for Otel record logs
+
 ## [0.6.3] - 2024-04-22
 
 ### Changed
@@ -223,7 +230,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - First release of SysFlow Processor.
 
-[Unreleased]: https://github.com/sysflow-telemetry/sf-processor/compare/0.6.3...HEAD
+[Unreleased]: https://github.com/sysflow-telemetry/sf-processor/compare/0.7.0...HEAD
+[0.7.0]: https://github.com/sysflow-telemetry/sf-processor/compare/0.6.3...0.7.0
 [0.6.3]: https://github.com/sysflow-telemetry/sf-processor/compare/0.6.2...0.6.3
 [0.6.2]: https://github.com/sysflow-telemetry/sf-processor/compare/0.6.1...0.6.2
 [0.6.1]: https://github.com/sysflow-telemetry/sf-processor/compare/0.6.0...0.6.1

Dockerfile

@@ -34,6 +34,8 @@ ENV GOPATH=/go/
 
 ENV SRC_ROOT=/go/src/github.com/sysflow-telemetry/sf-processor/
 
+ARG BACKEND_TAG=flatrecord
+
 # Install dependencies
 RUN dnf update -y --disableplugin=subscription-manager && \
      dnf install -y  --disableplugin=subscription-manager wget gcc make git device-mapper-devel
@@ -51,9 +53,10 @@ COPY makefile.manifest.inc ${SRC_ROOT}
 
 # Build
 RUN cd ${SRC_ROOT} && \
-    make SYSFLOW_VERSION=$VERSION \
-         SYSFLOW_BUILD_NUMBER=$BUILD_NUMBER \
-         install
+    make BACKEND_TAG=${BACKEND_TAG} \
+    SYSFLOW_VERSION=${VERSION} \
+    SYSFLOW_BUILD_NUMBER=${BUILD_NUMBER} \
+    install
 
 #-----------------------
 # Stage: runtime

Makefile

@@ -10,17 +10,18 @@
 include ./makefile.manifest.inc
 
 # Basic go commands
-PATH=$(shell printenv PATH):/usr/local/go/bin
-GOCMD=go
-GOBUILD=$(GOCMD) build -trimpath -tags "exclude_graphdriver_btrfs flatrecord"
-GOCLEAN=$(GOCMD) clean
-GOTEST=$(GOCMD) test -tags "exclude_graphdriver_btrfs flatrecord"
-GOGET=$(GOCMD) get -tags "exclude_graphdriver_btrfs flatrecord"
-BIN=sfprocessor
-OUTPUT=$(BIN)
-SRC=./driver
-PACKDIR=./scripts/cpack
-INSTALL_PATH=/usr/local/sysflow
+PATH = $(shell printenv PATH):/usr/local/go/bin
+BACKEND_TAG ?= flatrecord
+GOCMD = go
+GOBUILD = $(GOCMD) build -trimpath -tags "exclude_graphdriver_btrfs ${BACKEND_TAG}"
+GOCLEAN = $(GOCMD) clean
+GOTEST = $(GOCMD) test -tags "exclude_graphdriver_btrfs ${BACKEND_TAG}"
+GOGET = $(GOCMD) get -tags "exclude_graphdriver_btrfs ${BACKEND_TAG}"
+BIN = sfprocessor
+OUTPUT = $(BIN)
+SRC = ./driver
+PACKDIR = ./scripts/cpack
+INSTALL_PATH = /usr/local/sysflow
 
 .PHONY: build
 build: version deps
@@ -65,12 +66,15 @@ install: build
 	cp ./resources/policies/distribution/* /usr/local/sysflow/resources/policies/
 
 .PHONY: docker-build
-docker-build: docker-plugin-builder
-	( DOCKER_BUILDKIT=1 docker build --cache-from=sysflowtelemetry/plugin-builder:${SYSFLOW_VERSION} -t sysflowtelemetry/sf-processor:${SYSFLOW_VERSION} --build-arg UBI_VER=$(UBI_VERSION) --target=runtime -f Dockerfile . )
+docker-build: docker-plugin-builder	docker-processor
+
+.PHONY: docker-processor
+docker-processor:
+	( DOCKER_BUILDKIT=1 docker build -t sysflowtelemetry/sf-processor:${SYSFLOW_VERSION} --build-arg BACKEND_TAG=$(BACKEND_TAG) --build-arg UBI_VER=$(UBI_VERSION) --target=runtime -f Dockerfile . )
 
 .PHONY: docker-plugin-builder
 docker-plugin-builder:
-	( DOCKER_BUILDKIT=1 docker build -t sysflowtelemetry/plugin-builder:${SYSFLOW_VERSION} --build-arg UBI_VER=$(UBI_VERSION) --target=base -f Dockerfile . )
+	( DOCKER_BUILDKIT=1 docker build -t sysflowtelemetry/plugin-builder:${SYSFLOW_VERSION} --build-arg BACKEND_TAG=$(BACKEND_TAG) --build-arg UBI_VER=$(UBI_VERSION) --target=base -f Dockerfile . )
 
 .PHONY: pull
 pull:

README.md

@@ -6,7 +6,7 @@
 
 # Supported tags and respective `Dockerfile` links
 
--	[`0.6.3`, `latest`](https://github.com/sysflow-telemetry/sf-processor/blob/0.6.3/Dockerfile), [`edge`](https://github.com/sysflow-telemetry/sf-processor/blob/master/Dockerfile), [`dev`](https://github.com/sysflow-telemetry/sf-processor/blob/dev/Dockerfile)
+-	[`0.7.0-rc3`, `latest`](https://github.com/sysflow-telemetry/sf-processor/blob/0.7.0-rc3/Dockerfile), [`edge`](https://github.com/sysflow-telemetry/sf-processor/blob/master/Dockerfile), [`dev`](https://github.com/sysflow-telemetry/sf-processor/blob/dev/Dockerfile)
 
 # Quick reference
 
@@ -26,7 +26,7 @@
 	[docker hub](https://hub.docker.com/u/sysflowtelemetry) | [GHCR](https://github.com/orgs/sysflow-telemetry/packages)
 
 -	**Binary packages**:
-	[deb](https://github.com/sysflow-telemetry/sf-processor/releases/tag/0.6.3/sfprocessor-0.6.3-x86_64.deb) | [rpm](https://github.com/sysflow-telemetry/sf-processor/releases/tag/0.6.3/sfprocessor-0.6.3-x86_64.rpm) | [tgz](https://github.com/sysflow-telemetry/sf-processor/releases/tag/0.6.3/sfprocessor-0.6.3-x86_64.tar.gz)
+	[deb](https://github.com/sysflow-telemetry/sf-processor/releases/tag/0.7.0-rc3/sfprocessor-0.7.0-rc3-x86_64.deb) | [rpm](https://github.com/sysflow-telemetry/sf-processor/releases/tag/0.7.0-rc3/sfprocessor-0.7.0-rc3-x86_64.rpm) | [tgz](https://github.com/sysflow-telemetry/sf-processor/releases/tag/0.7.0-rc3/sfprocessor-0.7.0-rc3-x86_64.tar.gz)
 
 # What is SysFlow?
 

core/exporter/commons/config.go

@@ -58,7 +58,7 @@ type Config struct {
 	FileConfig
 	SyslogConfig
 	ESConfig
-	FindingsConfig
+	KafkaConfig
 }
 
 // CreateConfig creates a new config object from config dictionary.
@@ -125,7 +125,7 @@ func CreateConfig(conf map[string]interface{}) (c Config, err error) {
 	if err != nil {
 		return
 	}
-	c.FindingsConfig, err = CreateFindingsConfig(c, conf)
+	c.KafkaConfig, err = CreateKafkaConfig(c, conf)
 
 	return
 }
@@ -139,12 +139,12 @@ const (
 	FileTransport
 	SyslogTransport
 	ESTransport
-	FindingsTransport
+	KafkaTransport
 	NullTransport
 )
 
 func (s Transport) String() string {
-	return [...]string{"terminal", "file", "syslog", "es", "findings", "null"}[s]
+	return [...]string{"terminal", "file", "syslog", "es", "kafka", "null"}[s]
 }
 
 func parseTransportConfig(s string) Transport {
@@ -157,8 +157,8 @@ func parseTransportConfig(s string) Transport {
 	if ESTransport.String() == s {
 		return ESTransport
 	}
-	if FindingsTransport.String() == s {
-		return FindingsTransport
+	if KafkaTransport.String() == s {
+		return KafkaTransport
 	}
 	if NullTransport.String() == s {
 		return NullTransport
@@ -171,13 +171,13 @@ type Format int
 
 // Format config options.
 const (
-	JSONFormat       Format = iota // JSON schema
-	ECSFormat                      // Elastic Common Schema
-	OccurrenceFormat               // IBM Findings Occurrence
+	JSONFormat Format = iota // JSON schema
+	ECSFormat                // Elastic Common Schema
+	OtelFormat               // Open Telemetry schema
 )
 
 func (s Format) String() string {
-	return [...]string{"json", "ecs", "occurrence"}[s]
+	return [...]string{"json", "ecs", "otel"}[s]
 }
 
 func parseFormatConfig(s string) Format {
@@ -186,8 +186,8 @@ func parseFormatConfig(s string) Format {
 		return JSONFormat
 	case ECSFormat.String():
 		return ECSFormat
-	case OccurrenceFormat.String():
-		return OccurrenceFormat
+	case OtelFormat.String():
+		return OtelFormat
 	}
 	return JSONFormat
 }