chore: reorganise Package Structure (#97)

* publisher: log: add tests

* worker: test for ack function

* worker: tests: remove unused mocks

* worker: add tests for error cases

* worker: test: use constructor for tests

* app: worker: fix flush always timing out

* app: move kafka specific log messages to kafka publisher package

* wip: worker metrics test

* worker: add tests against instrumented metrics

* worker: rename timeSource to Clock

* chore: remove client example

* worker: use idiomatic null device for tests

* config: add tests for validation errors

* config: add test for valid configuration

* config: add tests for internal prepare()

* config: add tests for Walk()

* config: kafka: add tests for ToConfigMap()

* config: kafka: add more tests for config map serializer

* config: add tests for internal helper functions

* refactor: extract Clock to a stand alone package

* collector: add tests for TimePushed mutation

* serialization: add tests for SerialiseProto

* deserialization: add tests for DeserializeFunc

* worker: add test for flush time out

* config: refactor ack value names

* tests: integration e2e tests results in coverage data

* ci: test: fix coverage merge failing

* ci: prospective fix for coverdata #1

* ci: prospective fix for coverdata #2

* ci: prospective fix for coverdata #3

* ci: upload test coverage as artifact

* ci: refactor Docker build for release and testing

* services: grpc: add tests for synchronous event handling

* services: grpc: add tests to cover metric instrumentation

* services: rest: remove deadcode

For an incoming request, req.Body is never nil
https://pkg.go.dev/net/http#Request

* services: rest: add tests for error reading request body

* services: rest: add tests for malformed request body

* services: rest: refactor ack handling

* services: rest: add tests for different error paths

* ci: cache image pulls to speed up tests

* ci: fix docker image caching

* ci: ignore proto package during tests

* ci: remove docker image caching

* Revert "ci: ignore proto package during tests"

This reverts commit 75d0807.

* Revert "Revert "ci: ignore proto package during tests""

This reverts commit 1038f46.

* Revert "Revert "Revert "ci: ignore proto package during tests"""

This reverts commit a39cad6.

* publisher: log: move test event to root proto package

* ci: blacklist proto package during tests

* hack: publisher: log: use single spaced fields

* Revert "publisher: log: move test event to root proto package"

This reverts commit 7f837bf.

* Revert "ci: blacklist proto package during tests"

This reverts commit 0856073.

* Revert "hack: publisher: log: use single spaced fields"

This reverts commit 5ffb846.

* Revert "Revert "hack: publisher: log: use single spaced fields""

This reverts commit 67797d9.

* ci: ignore proto folders when computing coverage

* hack: publisher: log: use double spaced fields

* ci: fix coverage data merge

* ci: fix cover merge resulting in empty coverage file

* publish: kinesis: add tests for stream's existence check

* misc: improve test case description

* publisher: kinesis: add tests for hitting rate limits and quotas

* publisher: kinesis: refactor tests

* hygiene: reorganise package structure

* refactor: merge serialization and deserialization pkg into serde

* docs: fix broken dashboard link

* serde: test: fix broken references

* refactor: integrate middleware package into server/rest

app/proc.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/raystack/raccoon/config"
-	"github.com/raystack/raccoon/logger"
+	"github.com/raystack/raccoon/pkg/logger"
 )
 
 // Run the server

app/server.go

@@ -10,17 +10,17 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/raystack/raccoon/collector"
 	"github.com/raystack/raccoon/config"
-	"github.com/raystack/raccoon/logger"
-	"github.com/raystack/raccoon/metrics"
+	"github.com/raystack/raccoon/core/collector"
+	"github.com/raystack/raccoon/core/worker"
+	"github.com/raystack/raccoon/pkg/logger"
+	"github.com/raystack/raccoon/pkg/metrics"
 	"github.com/raystack/raccoon/publisher"
 	"github.com/raystack/raccoon/publisher/kafka"
 	"github.com/raystack/raccoon/publisher/kinesis"
 	logpub "github.com/raystack/raccoon/publisher/log"
 	"github.com/raystack/raccoon/publisher/pubsub"
-	"github.com/raystack/raccoon/services"
-	"github.com/raystack/raccoon/worker"
+	services "github.com/raystack/raccoon/server"
 
 	pubsubsdk "cloud.google.com/go/pubsub"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"

cmd/server.go

@@ -7,9 +7,8 @@ import (
 
 	"github.com/raystack/raccoon/app"
 	"github.com/raystack/raccoon/config"
-	"github.com/raystack/raccoon/logger"
-	"github.com/raystack/raccoon/metrics"
-	"github.com/raystack/raccoon/middleware"
+	"github.com/raystack/raccoon/pkg/logger"
+	"github.com/raystack/raccoon/pkg/metrics"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -24,7 +23,6 @@ func serverCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			middleware.Load()
 			metrics.Setup()
 			defer metrics.Close()
 			logger.SetLevel(config.Log.Level)

collector/collector.go → core/collector/collector.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"time"
 
-	"github.com/raystack/raccoon/identification"
+	"github.com/raystack/raccoon/core/identification"
 	pb "github.com/raystack/raccoon/proto"
 )
 

collector/service.go → core/collector/service.go

@@ -3,7 +3,7 @@ package collector
 import (
 	"context"
 
-	"github.com/raystack/raccoon/clock"
+	"github.com/raystack/raccoon/pkg/clock"
 )
 
 type ChannelCollector struct {

collector/service_test.go → core/collector/service_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/raystack/raccoon/clock"
+	"github.com/raystack/raccoon/pkg/clock"
 	"github.com/stretchr/testify/assert"
 )
 

deserialization/json.go → core/serde/json.go

@@ -1,7 +1,11 @@
-package deserialization
+package serde
 
 import "encoding/json"
 
 func DeserializeJSON(b []byte, i interface{}) error {
 	return json.Unmarshal(b, i)
 }
+
+func SerializeJSON(m interface{}) ([]byte, error) {
+	return json.Marshal(m)
+}

serialization/json_test.go → core/serde/json_test.go

@@ -1,4 +1,4 @@
-package serialization
+package serde
 
 import (
 	"reflect"
@@ -7,6 +7,38 @@ import (
 	pb "github.com/raystack/raccoon/proto"
 )
 
+func TestJSONDeserializer_Deserialize(t *testing.T) {
+	type args struct {
+		b []byte
+		i interface{}
+	}
+	tests := []struct {
+		name    string
+		j       DeserializeFunc
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Use JSON Deserializer",
+			j:    DeserializeJSON,
+			args: args{
+				b: []byte(`{"A": "a"}`),
+				i: &struct {
+					A string
+				}{},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.j(tt.args.b, tt.args.i); (err != nil) != tt.wantErr {
+				t.Errorf("JSONDeserializer.Deserialize() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func TestJSONSerializer_Serialize(t *testing.T) {
 	type args struct {
 		m interface{}

serialization/proto.go → core/serde/proto.go

@@ -1,4 +1,4 @@
-package serialization
+package serde
 
 import (
 	"errors"
@@ -17,3 +17,11 @@ func SerializeProto(m interface{}) ([]byte, error) {
 	}
 	return proto.Marshal(msg)
 }
+
+func DeserializeProto(b []byte, i interface{}) error {
+	msg, ok := i.(proto.Message)
+	if !ok {
+		return ErrInvalidProtoMessage
+	}
+	return proto.Unmarshal(b, msg)
+}

deserialization/proto_test.go → core/serde/proto_test.go

@@ -1,8 +1,9 @@
-package deserialization
+package serde
 
 import (
 	"testing"
 
+	"github.com/alecthomas/assert"
 	pb "github.com/raystack/raccoon/proto"
 )
 
@@ -44,3 +45,16 @@ func TestProtoDeserilizer_Deserialize(t *testing.T) {
 		})
 	}
 }
+
+func TestSerialiseProto(t *testing.T) {
+	t.Run("should return an error if argument is a non-protobuf message", func(t *testing.T) {
+		arg := struct{}{}
+		_, err := SerializeProto(arg)
+		assert.Equal(t, err, ErrInvalidProtoMessage)
+	})
+	t.Run("should serialize a proto message", func(t *testing.T) {
+		v := &pb.SendEventRequest{}
+		_, err := SerializeProto(v)
+		assert.Nil(t, err)
+	})
+}

core/serde/serde.go

@@ -0,0 +1,5 @@
+package serde
+
+type DeserializeFunc func(b []byte, i interface{}) error
+
+type SerializeFunc func(m interface{}) ([]byte, error)

worker/init_test.go → core/worker/init_test.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/raystack/raccoon/logger"
+	"github.com/raystack/raccoon/pkg/logger"
 )
 
 func TestMain(t *testing.M) {

worker/worker.go → core/worker/worker.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/raystack/raccoon/clock"
-	"github.com/raystack/raccoon/collector"
-	"github.com/raystack/raccoon/logger"
-	"github.com/raystack/raccoon/metrics"
+	"github.com/raystack/raccoon/core/collector"
+	"github.com/raystack/raccoon/pkg/clock"
+	"github.com/raystack/raccoon/pkg/logger"
+	"github.com/raystack/raccoon/pkg/metrics"
 	pb "github.com/raystack/raccoon/proto"
 	"github.com/raystack/raccoon/publisher"
 )

worker/worker_test.go → core/worker/worker_test.go

@@ -5,10 +5,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/raystack/raccoon/clock"
-	"github.com/raystack/raccoon/collector"
-	"github.com/raystack/raccoon/identification"
-	"github.com/raystack/raccoon/metrics"
+	"github.com/raystack/raccoon/core/collector"
+	"github.com/raystack/raccoon/core/identification"
+	"github.com/raystack/raccoon/pkg/clock"
+	"github.com/raystack/raccoon/pkg/metrics"
 	pb "github.com/raystack/raccoon/proto"
 	"github.com/raystack/raccoon/publisher"
 	"github.com/stretchr/testify/assert"

docs/docs/guides/monitoring.md

@@ -157,7 +157,7 @@ Raccoon provides fine-grained metrics that denote latency. That gives clues as t
 
 ### Dashboard
 
-There is a pre-built [grafana dashboard](https://github.com/raystack/raccoon/tree/dashboards/raccoon.json) available with support for Prometheus data source.
+There is a pre-built [grafana dashboard](https://github.com/raystack/raccoon/blob/main/grafana.json) available with support for Prometheus data source.
 
 If you're running the statsd + telegraf setup, you can configure telegraf to push metrics to Prometheus.
 

go.mod

@@ -5,6 +5,7 @@ go 1.22.4
 require (
 	cloud.google.com/go/pubsub v1.38.0
 	github.com/MakeNowJust/heredoc v1.0.0
+	github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
 	github.com/aws/aws-sdk-go-v2 v1.27.2
 	github.com/aws/aws-sdk-go-v2/config v1.27.18
 	github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.10
@@ -35,6 +36,8 @@ require (
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/iam v1.1.7 // indirect
 	github.com/alecthomas/chroma v0.8.2 // indirect
+	github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 // indirect
+	github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.18 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 // indirect
@@ -92,6 +95,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/schollz/progressbar/v3 v3.8.5 // indirect
+	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.16.0 // indirect

metrics/metrics.go → pkg/metrics/metrics.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 
 	"github.com/raystack/raccoon/config"
-	"github.com/raystack/raccoon/logger"
+	"github.com/raystack/raccoon/pkg/logger"
 )
 
 var instrument MetricInstrument = voidInstrument{}

metrics/prometheus.go → pkg/metrics/prometheus.go

@@ -10,7 +10,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/raystack/raccoon/config"
-	"github.com/raystack/raccoon/logger"
+	"github.com/raystack/raccoon/pkg/logger"
 	"github.com/spf13/cast"
 )
 

metrics/statsd.go → pkg/metrics/statsd.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/raystack/raccoon/config"
-	"github.com/raystack/raccoon/logger"
+	"github.com/raystack/raccoon/pkg/logger"
 	client "gopkg.in/alexcesaro/statsd.v2"
 )