diff --git a/api/api/openapi.yaml b/api/api/openapi.yaml index 551cc0a4..444c72a7 100644 --- a/api/api/openapi.yaml +++ b/api/api/openapi.yaml @@ -857,6 +857,13 @@ paths: schema: type: string style: form + - explode: true + in: query + name: wait + required: true + schema: + type: boolean + style: form responses: "204": description: No Content diff --git a/api/api_roles_discovery.go b/api/api_roles_discovery.go index 398187ef..1cb63aa4 100644 --- a/api/api_roles_discovery.go +++ b/api/api_roles_discovery.go @@ -970,6 +970,7 @@ type ApiDiscoverySubnetStartRequest struct { ctx context.Context ApiService *RolesDiscoveryApiService identifier *string + wait *bool } func (r ApiDiscoverySubnetStartRequest) Identifier(identifier string) ApiDiscoverySubnetStartRequest { @@ -977,6 +978,11 @@ func (r ApiDiscoverySubnetStartRequest) Identifier(identifier string) ApiDiscove return r } +func (r ApiDiscoverySubnetStartRequest) Wait(wait bool) ApiDiscoverySubnetStartRequest { + r.wait = &wait + return r +} + func (r ApiDiscoverySubnetStartRequest) Execute() (*http.Response, error) { return r.ApiService.DiscoverySubnetStartExecute(r) } @@ -1015,8 +1021,12 @@ func (a *RolesDiscoveryApiService) DiscoverySubnetStartExecute(r ApiDiscoverySub if r.identifier == nil { return nil, reportError("identifier is required and must be specified") } + if r.wait == nil { + return nil, reportError("wait is required and must be specified") + } parameterAddToHeaderOrQuery(localVarQueryParams, "identifier", r.identifier, "") + parameterAddToHeaderOrQuery(localVarQueryParams, "wait", r.wait, "") // to determine the Content-Type header localVarHTTPContentTypes := []string{} diff --git a/api/docs/RolesDiscoveryApi.md b/api/docs/RolesDiscoveryApi.md index e705c148..d889e84e 100644 --- a/api/docs/RolesDiscoveryApi.md +++ b/api/docs/RolesDiscoveryApi.md @@ -519,7 +519,7 @@ No authorization required ## DiscoverySubnetStart -> DiscoverySubnetStart(ctx).Identifier(identifier).Execute() +> DiscoverySubnetStart(ctx).Identifier(identifier).Wait(wait).Execute() Discovery Subnets @@ -537,10 +537,11 @@ import ( func main() { identifier := "identifier_example" // string | + wait := true // bool | configuration := openapiclient.NewConfiguration() apiClient := openapiclient.NewAPIClient(configuration) - r, err := apiClient.RolesDiscoveryApi.DiscoverySubnetStart(context.Background()).Identifier(identifier).Execute() + r, err := apiClient.RolesDiscoveryApi.DiscoverySubnetStart(context.Background()).Identifier(identifier).Wait(wait).Execute() if err != nil { fmt.Fprintf(os.Stderr, "Error when calling `RolesDiscoveryApi.DiscoverySubnetStart``: %v\n", err) fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) @@ -560,6 +561,7 @@ Other parameters are passed through a pointer to a apiDiscoverySubnetStartReques Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- **identifier** | **string** | | + **wait** | **bool** | | ### Return type diff --git a/pkg/extconfig/ip.go b/pkg/extconfig/ip.go index 17b5eea5..b457409e 100644 --- a/pkg/extconfig/ip.go +++ b/pkg/extconfig/ip.go @@ -63,7 +63,7 @@ func (e *ExtConfig) GetInterfaceForIP(forIp net.IP) (*net.Interface, error) { } } } - return nil, fmt.Errorf("faild to find interface for %s", forIp.String()) + return nil, fmt.Errorf("failed to find interface for %s", forIp.String()) } func (e *ExtConfig) GetIP() (net.IP, error) { diff --git a/pkg/roles/dhcp/role_test.go b/pkg/roles/dhcp/role_test.go index b27593ea..55cea65b 100644 --- a/pkg/roles/dhcp/role_test.go +++ b/pkg/roles/dhcp/role_test.go @@ -14,6 +14,7 @@ import ( func generateHW() net.HardwareAddr { return net.HardwareAddr(securecookie.GenerateRandomKey(6)) } + func RoleConfig() []byte { return []byte(tests.MustJSON(dhcp.RoleConfig{ Port: 0, diff --git a/pkg/roles/discovery/api_subnets.go b/pkg/roles/discovery/api_subnets.go index ebb83ec9..3d0afd0d 100644 --- a/pkg/roles/discovery/api_subnets.go +++ b/pkg/roles/discovery/api_subnets.go @@ -87,6 +87,7 @@ func (r *Role) APISubnetsPut() usecase.Interactor { type APISubnetsStartInput struct { Name string `query:"identifier" required:"true"` + Wait bool `query:"wait" required:"true"` } func (r *Role) APISubnetsStart() usecase.Interactor { @@ -107,7 +108,11 @@ func (r *Role) APISubnetsStart() usecase.Interactor { r.log.Warn("failed to parse subnet from KV", zap.Error(err)) return status.Wrap(err, status.Internal) } - go s.RunDiscovery(context.Background()) + if input.Wait { + s.RunDiscovery(context.Background()) + } else { + go s.RunDiscovery(context.Background()) + } return nil }) u.SetName("discovery.subnet_start") diff --git a/pkg/roles/discovery/device.go b/pkg/roles/discovery/device.go index 201d5a29..dbb788c5 100644 --- a/pkg/roles/discovery/device.go +++ b/pkg/roles/discovery/device.go @@ -10,7 +10,6 @@ import ( dhcptypes "beryju.io/gravity/pkg/roles/dhcp/types" "beryju.io/gravity/pkg/roles/discovery/types" dnstypes "beryju.io/gravity/pkg/roles/dns/types" - "github.com/google/uuid" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" @@ -27,8 +26,7 @@ type Device struct { func (r *Role) newDevice() *Device { return &Device{ - Identifier: uuid.New().String(), - inst: r.i, + inst: r.i, } } diff --git a/pkg/roles/discovery/role.go b/pkg/roles/discovery/role.go index fc3f1e78..1699b9df 100644 --- a/pkg/roles/discovery/role.go +++ b/pkg/roles/discovery/role.go @@ -2,7 +2,9 @@ package discovery import ( "context" - "net/netip" + "errors" + "fmt" + "net" "beryju.io/gravity/pkg/extconfig" instanceTypes "beryju.io/gravity/pkg/instance/types" @@ -63,14 +65,14 @@ func New(instance roles.Instance) *Role { }) r.i.AddEventListener(instanceTypes.EventTopicInstanceFirstStart, func(ev *roles.Event) { // On first start create a subnet based on the instance IP - subnet := r.NewSubnet("default-instance-subnet") - ip := netip.MustParseAddr(extconfig.Get().Instance.IP) - prefix, err := ip.Prefix(24) + subnet := r.NewSubnet(fmt.Sprintf("instance-subnet-%s", extconfig.Get().Instance.Identifier)) + + cidr, err := GetCIDRFromIP() if err != nil { r.log.Warn("failed to get prefix", zap.Error(err)) return } - subnet.CIDR = prefix.String() + subnet.CIDR = cidr subnet.DNSResolver = extconfig.Get().FallbackDNS subnet.DiscoveryTTL = 86400 err = subnet.put(ev.Context) @@ -83,6 +85,28 @@ func New(instance roles.Instance) *Role { return r } +func GetCIDRFromIP() (string, error) { + ip := net.ParseIP(extconfig.Get().Instance.IP) + intf, err := extconfig.Get().GetInterfaceForIP(ip) + if err != nil { + return "", err + } + addrs, err := intf.Addrs() + if err != nil { + return "", err + } + for _, addr := range addrs { + iip, net, err := net.ParseCIDR(addr.String()) + if err != nil { + continue + } + if ip.Equal(iip) { + return net.String(), nil + } + } + return "", errors.New("no CIDR found") +} + func (r *Role) Start(ctx context.Context, config []byte) error { r.cfg = r.decodeRoleConfig(config) if !r.cfg.Enabled || extconfig.Get().ListenOnlyMode { diff --git a/pkg/roles/discovery/subnet.go b/pkg/roles/discovery/subnet.go index 288fde87..51ba1fd3 100644 --- a/pkg/roles/discovery/subnet.go +++ b/pkg/roles/discovery/subnet.go @@ -125,8 +125,12 @@ func (s *Subnet) RunDiscovery(ctx context.Context) []Device { dev.MAC = addr.Addr } else { dev.IP = addr.Addr + dev.Identifier = addr.Addr } } + if dev.Identifier == "" { + continue + } devices = append(devices, *dev) err := dev.put(tr.Context(), int64(s.DiscoveryTTL)) if err != nil { diff --git a/pkg/roles/discovery/subnet_test.go b/pkg/roles/discovery/subnet_test.go index f193b978..83c9d417 100644 --- a/pkg/roles/discovery/subnet_test.go +++ b/pkg/roles/discovery/subnet_test.go @@ -1,52 +1,179 @@ package discovery_test import ( + "net" "testing" - "beryju.io/gravity/pkg/extconfig" "beryju.io/gravity/pkg/instance" + "beryju.io/gravity/pkg/roles/dhcp" + dhcpTypes "beryju.io/gravity/pkg/roles/dhcp/types" "beryju.io/gravity/pkg/roles/discovery" + "beryju.io/gravity/pkg/roles/discovery/types" + "beryju.io/gravity/pkg/roles/dns" + dnsTypes "beryju.io/gravity/pkg/roles/dns/types" "beryju.io/gravity/pkg/tests" + "github.com/gorilla/securecookie" "github.com/stretchr/testify/assert" ) const ( DockerNetworkCIDR = "10.200.0.0/28" - - DockerIPCoreDNS = "10.200.0.4" ) -func TestDiscoveryDocker(t *testing.T) { +func TestDiscoveryConvert(t *testing.T) { defer tests.Setup(t)() - if !tests.HasLocalDocker() { - return - } - extconfig.Get().ListenOnlyMode = false rootInst := instance.New() ctx := tests.Context() - inst := rootInst.ForRole("discovery", ctx) - role := discovery.New(inst) + role := discovery.New(rootInst.ForRole("discovery", ctx)) assert.NotNil(t, role) assert.Nil(t, role.Start(ctx, []byte(tests.MustJSON(discovery.RoleConfig{ Enabled: true, })))) defer role.Stop() - sub := role.NewSubnet("docker-test") - sub.CIDR = DockerNetworkCIDR - sub.DNSResolver = DockerIPCoreDNS - devices := sub.RunDiscovery(ctx) - assert.Equal(t, "10.200.0.1", devices[0].IP) + inst := rootInst.ForRole("test", ctx) + + // Create DNS Zone to register host in + tests.PanicIfError(inst.KV().Put( + ctx, + inst.KV().Key( + dnsTypes.KeyRole, + dnsTypes.KeyZones, + "example.com.", + ).String(), + tests.MustJSON(dns.Zone{ + HandlerConfigs: []map[string]interface{}{ + { + "type": "etcd", + }, + }, + }), + )) + // Create DNS Zone for reverse + tests.PanicIfError(inst.KV().Put( + ctx, + inst.KV().Key( + dnsTypes.KeyRole, + dnsTypes.KeyZones, + "200.10.in-addr.arpa.", + ).String(), + tests.MustJSON(dns.Zone{ + HandlerConfigs: []map[string]interface{}{ + { + "type": "etcd", + }, + }, + }), + )) + + // Create DHCP Scope to register host in + tests.PanicIfError(inst.KV().Put( + ctx, + inst.KV().Key( + dhcpTypes.KeyRole, + dhcpTypes.KeyScopes, + "test", + ).String(), + tests.MustJSON(dhcp.Scope{ + SubnetCIDR: DockerNetworkCIDR, + TTL: 86400, + DNS: &dhcp.ScopeDNS{ + Zone: "example.com.", + }, + IPAM: map[string]string{ + "type": "internal", + "range_start": "10.200.0.1", + "range_end": "10.200.0.250", + }, + }), + )) + + // Start DNS & DHCP to register events + dnsr := dns.New(rootInst.ForRole("dns", ctx)) + assert.NotNil(t, dnsr) + assert.Nil(t, dnsr.Start(ctx, []byte(tests.MustJSON(dns.RoleConfig{ + Port: -1, + })))) + defer dnsr.Stop() + + dhcpr := dhcp.New(rootInst.ForRole("dhcp", ctx)) + assert.NotNil(t, dhcpr) + assert.Nil(t, dhcpr.Start(ctx, []byte(tests.MustJSON(dhcp.RoleConfig{ + Port: -1, + })))) + defer dhcpr.Stop() - assert.Equal(t, "etcd.t.gravity.beryju.io", devices[1].Hostname) - assert.Equal(t, "10.200.0.2", devices[1].IP) - assert.Equal(t, "", devices[1].MAC) + // Manually create a device + mac := net.HardwareAddr(securecookie.GenerateRandomKey(6)).String() + tests.PanicIfError(inst.KV().Put( + ctx, + inst.KV().Key( + types.KeyRole, + types.KeyDevices, + "test", + ).String(), + tests.MustJSON(discovery.Device{ + IP: "10.200.0.1", + Hostname: "foo", + MAC: mac, + }), + )) - assert.Equal(t, "minio.t.gravity.beryju.io", devices[2].Hostname) - assert.Equal(t, "10.200.0.3", devices[2].IP) - assert.Equal(t, "", devices[2].MAC) + err := role.APIDevicesApply().Interact(ctx, discovery.APIDevicesApplyInput{ + Identifier: "test", + To: "dhcp", + DHCPScope: "test", + DNSZone: "example.com.", + }, &struct{}{}) + assert.NoError(t, err) - assert.Equal(t, "coredns.t.gravity.beryju.io", devices[3].Hostname) - assert.Equal(t, "10.200.0.4", devices[3].IP) - assert.Equal(t, "", devices[3].MAC) + // Check DHCP lease + tests.AssertEtcd( + t, + inst.KV(), + inst.KV().Key( + dhcpTypes.KeyRole, + dhcpTypes.KeyLeases, + mac, + ), + dhcp.Lease{ + ScopeKey: "test", + Address: "10.200.0.1", + Hostname: "foo", + Expiry: 0, + Description: "", + }, + ) + // Check forward DNS Record + tests.AssertEtcd( + t, + inst.KV(), + inst.KV().Key( + dnsTypes.KeyRole, + dnsTypes.KeyZones, + "example.com.", + "foo", + "A", + mac, + ), + dns.Record{ + Data: "10.200.0.1", + }, + ) + // Check reverse DNS Record + tests.AssertEtcd( + t, + inst.KV(), + inst.KV().Key( + dnsTypes.KeyRole, + dnsTypes.KeyZones, + "200.10.in-addr.arpa.", + "1.0", + "PTR", + mac, + ), + dns.Record{ + Data: "foo.example.com.", + }, + ) } diff --git a/pkg/tests/utils.go b/pkg/tests/utils.go index c8d7b4ec..46592bd8 100644 --- a/pkg/tests/utils.go +++ b/pkg/tests/utils.go @@ -110,10 +110,6 @@ func Setup(t *testing.T) func() { } } -func HasLocalDocker() bool { - return runtime.GOOS == "linux" -} - func Listen(port int32) string { if runtime.GOOS == "darwin" { return fmt.Sprintf(":%d", port) diff --git a/schema.yml b/schema.yml index 195b9b44..479f5516 100644 --- a/schema.yml +++ b/schema.yml @@ -794,6 +794,11 @@ paths: required: true schema: type: string + - in: query + name: wait + required: true + schema: + type: boolean responses: "204": description: No Content diff --git a/tests/discovery_e2e_test.go b/tests/discovery_e2e_test.go new file mode 100644 index 00000000..691baef5 --- /dev/null +++ b/tests/discovery_e2e_test.go @@ -0,0 +1,63 @@ +package tests + +import ( + "testing" + + "beryju.io/gravity/tests/gravity" + dockernetwork "github.com/docker/docker/api/types/network" + "github.com/stretchr/testify/assert" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/network" +) + +func TestDiscovery_Simple(t *testing.T) { + ctx := Context(t) + + net, err := network.New( + ctx, + network.WithIPAM(&dockernetwork.IPAM{ + Driver: "default", + Config: []dockernetwork.IPAMConfig{ + { + Subnet: "10.100.0.0/29", + }, + }, + }), + network.WithAttachable(), + ) + assert.NoError(t, err) + testcontainers.CleanupNetwork(t, net) + + gr := gravity.New(t, + gravity.WithNet(net), + // Use Docker gateway as DNS server to lookup name of other containers + gravity.WithEnv("FALLBACK_DNS", "10.100.0.1:53")) + + tester, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../hack/e2e/", + Dockerfile: "dns.Dockerfile", + Repo: "gravity-dns-client", + KeepImage: true, + }, + Hostname: "client", + Networks: []string{net.Name}, + }, + Started: true, + }) + testcontainers.CleanupContainer(t, tester) + assert.NoError(t, err) + + _, err = gr.APIClient().RolesDiscoveryApi. + DiscoverySubnetStart(ctx). + Identifier("instance-subnet-gravity-1"). + Wait(true). + Execute() + assert.NoError(t, err) + + d, _, err := gr.APIClient().RolesDiscoveryApi.DiscoveryGetDevices(ctx).Execute() + assert.NoError(t, err) + + assert.Len(t, d.Devices, 3) +} diff --git a/tests/dns_e2e_test.go b/tests/dns_e2e_test.go index cece93fc..34ee92a9 100644 --- a/tests/dns_e2e_test.go +++ b/tests/dns_e2e_test.go @@ -12,7 +12,7 @@ func TestDNS_SimpleDefault(t *testing.T) { ctx := Context(t) gravity.New(t) - // DHCP tester + // DNS tester tester, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ FromDockerfile: testcontainers.FromDockerfile{ diff --git a/tests/go.mod b/tests/go.mod index cfdfa366..f354e4c0 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -84,3 +84,5 @@ require ( google.golang.org/protobuf v1.36.3 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace beryju.io/gravity => ../