tests: even more tests (#1449)

* mirror structure for server target

* remove unused function

* start testing some CLI stuff

* improve logs

* add dhcp relay test

* more logging

* add utils to dhcp images

* add gravity options

* re-org code

* test different workaround

* cleanup

* fix opts not applying

* dont check getenv all the time

* fix paths

* make makefile better

* dhcp will always be udp

Makefile

@@ -12,6 +12,9 @@ SCHEMA_FILE = schema.yml
 TEST_COUNT = 1
 TEST_FLAGS =
 
+GEN_API_TS = "gen-ts-api"
+GEN_API_GO = "api"
+
 ci--env:
 	echo "sha=${GITHUB_SHA}" >> ${GITHUB_OUTPUT}
 	echo "build=${GITHUB_RUN_ID}" >> ${GITHUB_OUTPUT}
@@ -22,7 +25,7 @@ docker-build: internal/resources/macoui internal/resources/blocky internal/resou
 	go build \
 		-ldflags "${LD_FLAGS} -X beryju.io/gravity/pkg/extconfig.BuildHash=${GIT_BUILD_HASH}" \
 		${GO_BUILD_FLAGS} \
-		-v -a -o gravity ${PWD}
+		-v -a -o gravity ${PWD}/cmd/server/main
 
 clean:
 	rm -rf ${PWD}/data/
@@ -34,7 +37,10 @@ run: internal/resources/macoui internal/resources/blocky internal/resources/tftp
 	export DEBUG=true
 	export LISTEN_ONLY=true
 	$(eval LD_FLAGS := -X beryju.io/gravity/pkg/extconfig.Version=${VERSION} -X beryju.io/gravity/pkg/extconfig.BuildHash=dev-$(shell git rev-parse HEAD))
-	go run ${GO_FLAGS} ${PWD} server
+	go run \
+		${GO_FLAGS} \
+		${PWD}/cmd/server/main \
+			server
 
 # Web
 web: web-lint web-build
@@ -90,21 +96,25 @@ internal/resources/tftp:
 	curl -L https://boot.netboot.xyz/ipxe/netboot.xyz.efi -o ${PWD}/internal/resources/tftp/netboot.xyz.efi
 
 gen-build:
-	DEBUG=true go run ${GO_FLAGS} ${PWD} generateSchema ${SCHEMA_FILE}
+	export DEBUG=true
+	go run \
+		${GO_FLAGS} \
+		${PWD}/cmd/server/main \
+			generateSchema ${SCHEMA_FILE}
 	git add ${SCHEMA_FILE}
 
 gen-proto:
 	protoc \
 		--proto_path . \
-		--go_out . \
+		--go_out ${PWD} \
 		protobuf/**
 
 gen-clean:
-	rm -rf ${PWD}/gen-ts-api/
-	rm -rf ${PWD}/api/api/
-	rm -rf ${PWD}/api/docs/
-	rm -rf ${PWD}/api/test/
-	rm -rf ${PWD}/api/*.go
+	rm -rf ${PWD}/${GEN_API_TS}/
+	rm -rf ${PWD}/${GEN_API_GO}/api/
+	rm -rf ${PWD}/${GEN_API_GO}/docs/
+	rm -rf ${PWD}/${GEN_API_GO}/test/
+	rm -rf ${PWD}/${GEN_API_GO}/*.go
 
 gen-tag:
 	git add Makefile
@@ -123,15 +133,15 @@ gen-client-go:
 		--additional-properties=packageName=api \
 		-i /local/schema.yml \
 		-g go \
-		-o /local/api \
-		-c /local/api/config.yaml
-	cd ${PWD}/api/
+		-o /local/${GEN_API_GO} \
+		-c /local/${GEN_API_GO}/config.yaml
+	cd ${PWD}/${GEN_API_GO}/
 	rm -f .travis.yml go.mod go.sum
 	go get
-	go fmt .
+	go fmt ${PWD}/${GEN_API_GO}/
 	go mod tidy
-	gofumpt -l -w . || true
-	git add .
+	gofumpt -l -w ${PWD}/${GEN_API_GO}/ || true
+	git add ${PWD}/${GEN_API_GO}/
 
 gen-client-ts:
 	docker run \
@@ -140,15 +150,15 @@ gen-client-ts:
 		openapitools/openapi-generator-cli:v6.6.0 generate \
 		-i /local/${SCHEMA_FILE} \
 		-g typescript-fetch \
-		-o /local/gen-ts-api \
+		-o /local/${GEN_API_TS} \
 		--additional-properties=typescriptThreePlus=true,supportsES6=true,npmName=gravity-api,npmVersion=${VERSION} \
 		--git-repo-id BeryJu \
 		--git-user-id gravity
-	cd ${PWD}/gen-ts-api && npm i
-	\cp -rf ${PWD}/gen-ts-api/* ${PWD}/web/node_modules/gravity-api
+	cd ${PWD}/${GEN_API_TS} && npm i
+	\cp -rf ${PWD}/${GEN_API_TS}/* ${PWD}/web/node_modules/gravity-api
 
 gen-client-ts-publish: gen-client-ts
-	cd ${PWD}/gen-ts-api
+	cd ${PWD}/${GEN_API_TS}
 	npm publish
 	cd ${PWD}/web
 	npm i gravity-api@${VERSION}

api/.openapi-generator/FILES

@@ -226,15 +226,4 @@ model_types_api_metrics_role.go
 model_types_dhcp_option.go
 model_types_oidc_config.go
 response.go
-test/api_cluster_instances_test.go
-test/api_cluster_test.go
-test/api_roles_api_test.go
-test/api_roles_backup_test.go
-test/api_roles_dhcp_test.go
-test/api_roles_discovery_test.go
-test/api_roles_dns_test.go
-test/api_roles_etcd_test.go
-test/api_roles_monitoring_test.go
-test/api_roles_tftp_test.go
-test/api_roles_tsdb_test.go
 utils.go

cmd/cli/cli_health.go

@@ -1,21 +1,33 @@
 package cli
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
+	"go.uber.org/zap"
 )
 
 var healthCmd = &cobra.Command{
 	Use:   "health",
 	Short: "Check health and version",
 	Run: func(cmd *cobra.Command, args []string) {
-		v, hr, err := apiClient.ClusterInstancesApi.ClusterGetInstanceInfo(cmd.Context()).Execute()
+		c, hr, err := apiClient.ClusterApi.ClusterGetClusterInfo(cmd.Context()).Execute()
 		if err != nil {
 			checkApiError(hr, err)
 			os.Exit(1)
 		}
-		logger.Info(v.Version)
+		m := map[string]interface{}{
+			"clusterVersion": c.ClusterVersion,
+			"instances":      c.Instances,
+		}
+		b, err := json.MarshalIndent(m, "", "\t")
+		if err != nil {
+			logger.Warn("failed to render JSON", zap.Error(err))
+			return
+		}
+		fmt.Println(string(b))
 	},
 }
 

cmd/generateSchema.go → cmd/server/generateSchema.go

@@ -1,4 +1,4 @@
-package cmd
+package server
 
 import (
 	"context"
@@ -69,6 +69,8 @@ var generateSchemaCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(generateSchemaCmd)
+	if extconfig.Get().Debug {
+		rootCmd.AddCommand(generateSchemaCmd)
+	}
 	generateSchemaCmd.PersistentFlags().StringVarP(&schemaFormat, "format", "f", "yaml", "Output format (yaml/json)")
 }

cmd/generateSchema_test.go → cmd/server/generateSchema_test.go

@@ -1,18 +1,18 @@
-package cmd_test
+package server_test
 
 import (
 	"encoding/json"
 	"testing"
 
-	"beryju.io/gravity/cmd"
+	"beryju.io/gravity/cmd/server"
 	"beryju.io/gravity/pkg/tests"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestGenerateSchema(t *testing.T) {
 	defer tests.Setup(t)()
 	called := false
-	cmd.GenerateSchema(tests.Context(), "json", func(schema []byte) {
+	server.GenerateSchema(tests.Context(), "json", func(schema []byte) {
 		assert.NotEqual(t, "", string(schema))
 		var out interface{}
 		err := json.Unmarshal(schema, &out)

cmd/server/main/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"beryju.io/gravity/cmd/server"
+)
+
+func main() {
+	server.Execute()
+}

cmd/root.go → cmd/server/root.go

@@ -1,4 +1,4 @@
-package cmd
+package server
 
 import (
 	"os"

cmd/server.go → cmd/server/server.go

@@ -1,4 +1,4 @@
-package cmd
+package server
 
 import (
 	"os"

hack/e2e/dhcp-client.Dockerfile

@@ -1,7 +1,10 @@
 FROM docker.io/library/ubuntu:24.04
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update && \
-    apt-get install -y iproute2 isc-dhcp-client tcpdump
+    apt-get install -y --no-install-recommends iproute2 isc-dhcp-client tcpdump && \
+    apt-get clean
 
 COPY ./dhcp-client/entrypoint.sh /entrypoint.sh
 

hack/e2e/dhcp-relay.Dockerfile

@@ -1,9 +1,9 @@
-FROM library/ubuntu:24.04
+FROM docker.io/library/ubuntu:24.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends isc-dhcp-relay && \
+    apt-get install -y --no-install-recommends isc-dhcp-relay iproute2 tcpdump && \
     apt-get clean
 
 STOPSIGNAL SIGINT

pkg/roles/dhcp/dhcp_handler4.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 	"sync"
 
@@ -16,6 +17,15 @@ import (
 	"golang.org/x/net/ipv4"
 )
 
+func getIP(addr net.Addr) net.IP {
+	clientIP := ""
+	switch addr := addr.(type) {
+	case *net.UDPAddr:
+		clientIP = addr.IP.String()
+	}
+	return net.ParseIP(clientIP)
+}
+
 var ErrNilResponse = errors.New("no DHCP response")
 
 // Credit to CoreDHCP
@@ -51,6 +61,12 @@ func (h *handler4) Serve() error {
 	}
 }
 
+var debugDHCPGatewayReplyPeer bool
+
+func init() {
+	debugDHCPGatewayReplyPeer = os.Getenv("GRAVITY_DEBUG_DHCP_GATEWAY_REPLY_CIADDR") != ""
+}
+
 func (h *handler4) Handle(buf []byte, oob *ipv4.ControlMessage, peer net.Addr) error {
 	if extconfig.Get().ListenOnlyMode {
 		return nil
@@ -94,12 +110,24 @@ func (h *handler4) Handle(buf []byte, oob *ipv4.ControlMessage, peer net.Addr) e
 	useEthernet := false
 	var p *net.UDPAddr
 	if !r.GatewayIPAddr.IsUnspecified() {
+		r.log.Debug("sending response to gateway")
+		// giaddr should be set to the Relay's IP Address, however it is the IP of the subnet
+		// the client should get an IP for. We might not always be able to directly reply to that IP
+		// especially in environments where we can't adjust firewall/routing rules. (like e2e tests)
+		// when this environment variable is set, reply directly to the IP we got the UDP request from,
+		// which is not the RFC defined behaviour
 		p = &net.UDPAddr{IP: r.GatewayIPAddr, Port: dhcpv4.ServerPort}
+		if debugDHCPGatewayReplyPeer {
+			p.IP = getIP(r.peer)
+		}
 	} else if resp.MessageType() == dhcpv4.MessageTypeNak {
+		r.log.Debug("sending response to bcast (NAK)")
 		p = &net.UDPAddr{IP: net.IPv4bcast, Port: dhcpv4.ClientPort}
 	} else if !r.ClientIPAddr.IsUnspecified() {
+		r.log.Debug("sending response to client")
 		p = &net.UDPAddr{IP: r.ClientIPAddr, Port: dhcpv4.ClientPort}
 	} else if r.IsBroadcast() {
+		r.log.Debug("sending response to bcast")
 		p = &net.UDPAddr{IP: net.IPv4bcast, Port: dhcpv4.ClientPort}
 	} else {
 		// sends a layer2 frame so that we can define the destination MAC address

pkg/roles/dhcp/dhcp_handler4_discover.go

@@ -20,13 +20,13 @@ func (r *Role) HandleDHCPDiscover4(req *Request4) *dhcpv4.DHCPv4 {
 		}
 		err := match.Put(req.Context, int64(r.cfg.LeaseNegotiateTimeout))
 		if err != nil {
-			req.log.Warn("failed to update lease", zap.Error(err))
+			req.log.Warn("failed to update lease during discover creation", zap.Error(err))
 		}
 	} else {
 		go func() {
 			err := match.Put(req.Context, match.scope.TTL)
 			if err != nil {
-				req.log.Warn("failed to update lease", zap.Error(err))
+				req.log.Warn("failed to update lease during discover", zap.Error(err))
 			}
 		}()
 	}

pkg/roles/dhcp/leases.go

@@ -58,7 +58,7 @@ func (r *Role) FindLease(req *Request4) *Lease {
 		go func() {
 			err := lease.Put(req.Context, lease.scope.TTL)
 			if err != nil {
-				r.log.Warn("failed to update lease", zap.Error(err))
+				r.log.Warn("failed to update lease for re-assigned IP", zap.Error(err))
 			}
 		}()
 	}
@@ -228,7 +228,7 @@ func (l *Lease) createReply(req *Request4) *dhcpv4.DHCPv4 {
 		go func() {
 			err := l.Put(req.Context, l.Expiry)
 			if err != nil {
-				l.log.Warn("failed to update lease", zap.Error(err))
+				l.log.Warn("failed to update lease for updated hostname", zap.Error(err))
 			}
 		}()
 	}

pkg/roles/dhcp/role.go

@@ -147,7 +147,6 @@ func (r *Role) initServer4() error {
 		IP:   net.ParseIP("0.0.0.0"),
 		Port: r.cfg.Port,
 	}
-	var err error
 	ifName := extconfig.Get().Instance.Interface
 	udpConn, err := server4.NewIPv4UDPConn(ifName, laddr)
 	if err != nil {

pkg/tests/utils.go

@@ -4,12 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net"
 	"net/netip"
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"beryju.io/gravity/pkg/extconfig"
 	"beryju.io/gravity/pkg/storage"
@@ -122,24 +120,3 @@ func Listen(port int32) string {
 	}
 	return extconfig.Get().Listen(port)
 }
-
-func WaitForPort(port int32) {
-	max := 30
-	try := 0
-	listen := Listen(port)
-	time.Sleep(500 * time.Millisecond)
-	for {
-		ln, err := net.Listen("tcp", listen)
-		if ln != nil {
-			_ = ln.Close()
-		}
-		if err != nil {
-			return
-		}
-		try += 1
-		if try >= max {
-			panic(fmt.Errorf("failed to wait for port '%s' to be listening", listen))
-		}
-		time.Sleep(1 * time.Millisecond)
-	}
-}