Skip to content

Commit

Permalink
Improve engine tracking of configured VIP addresses.
Browse files Browse the repository at this point in the history 
Otherwise the engine can leave configured (and try to act upon) no
longer used VIP addresses.

- When receiving a vserver config update, check that the previously
  configured unicast VIP addresses are still needed, and remove them if
  not.
- Better document relevant vserver fields.
- Ensure vips and lbVservers maps are in-sync with the interface by
  pruning as necessary.
- Implement a stub lb interface for testing that allows introspection of
  live configuration.
- Use it to validate state during a re-ip transition.
- Other minor test style cleanup.
  • Loading branch information
@baptr
baptr committed May 6, 2016
1 parent 7c1493a commit de11b28
Show file tree
Hide file tree
Showing 9 changed files with 379 additions and 53 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ After setting `GOPATH` to an appropriate location (for example `~/go`):
go get -u github.com/golang/glog
go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
go get -u github.com/miekg/dns
go get -u github.com/kylelemons/godebug/pretty

Ensure that `${GOPATH}/bin` is in your `${PATH}` and in the seesaw directory:

Expand Down
79 changes: 68 additions & 11 deletions engine/engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@ package engine
// engine tests.

import (
"fmt"
"net"

"github.com/google/seesaw/common/seesaw"
"github.com/google/seesaw/ipvs"
ncclient "github.com/google/seesaw/ncc/client"
ncctypes "github.com/google/seesaw/ncc/types"
"github.com/google/seesaw/quagga"

log "github.com/golang/glog"
)

type dummyNCC struct{}
Expand All @@ -53,22 +56,76 @@ func (nc *dummyNCC) IPVSUpdateDestination(svc *ipvs.Service, dst *ipvs.Destinati
func (nc *dummyNCC) IPVSDeleteDestination(svc *ipvs.Service, dst *ipvs.Destination) error { return nil }
func (nc *dummyNCC) RouteDefaultIPv4() (net.IP, error) { return nil, nil }

type dummyLBInterface struct{}
type dummyLBInterface struct {
vips map[seesaw.VIP]bool
vlans map[uint16]bool
vservers map[string]map[seesaw.AF]bool
}

func newDummyLBInterface() *dummyLBInterface {
return &dummyLBInterface{
make(map[seesaw.VIP]bool),
make(map[uint16]bool),
make(map[string]map[seesaw.AF]bool),
}
}

func (lb *dummyLBInterface) Init() error { return nil }
func (lb *dummyLBInterface) Up() error { return nil }
func (lb *dummyLBInterface) Down() error { return nil }
func (lb *dummyLBInterface) AddVIP(vip *seesaw.VIP) error { return nil }
func (lb *dummyLBInterface) DeleteVIP(vip *seesaw.VIP) error { return nil }
func (lb *dummyLBInterface) AddVLAN(vlan *seesaw.VLAN) error { return nil }
func (lb *dummyLBInterface) DeleteVLAN(vlan *seesaw.VLAN) error { return nil }
func (lb *dummyLBInterface) AddVserver(v *seesaw.Vserver, af seesaw.AF) error { return nil }
func (lb *dummyLBInterface) DeleteVserver(v *seesaw.Vserver, af seesaw.AF) error { return nil }
func (lb *dummyLBInterface) Init() error { return nil }
func (lb *dummyLBInterface) Up() error { return nil }
func (lb *dummyLBInterface) Down() error { return nil }
func (lb *dummyLBInterface) AddVIP(vip *seesaw.VIP) error {
log.Infof("Adding vip %v", vip)
lb.vips[*vip] = true
return nil
}
func (lb *dummyLBInterface) DeleteVIP(vip *seesaw.VIP) error {
log.Infof("Deleting vip %v", vip)
if _, ok := lb.vips[*vip]; !ok {
return fmt.Errorf("deleting non-existent VIP: %v", vip)
}
delete(lb.vips, *vip)
return nil
}
func (lb *dummyLBInterface) AddVLAN(vlan *seesaw.VLAN) error {
lb.vlans[vlan.Key()] = true
return nil
}
func (lb *dummyLBInterface) DeleteVLAN(vlan *seesaw.VLAN) error {
if _, ok := lb.vlans[vlan.Key()]; !ok {
return fmt.Errorf("deleting non-existent VLAN: %v", vlan)
}
delete(lb.vlans, vlan.Key())
return nil
}
func (lb *dummyLBInterface) AddVserver(v *seesaw.Vserver, af seesaw.AF) error {
log.Infof("Adding vserver %v", v)
afMap, ok := lb.vservers[v.Name]
if !ok {
afMap = make(map[seesaw.AF]bool)
lb.vservers[v.Name] = afMap
}
afMap[af] = true
return nil
}
func (lb *dummyLBInterface) DeleteVserver(v *seesaw.Vserver, af seesaw.AF) error {
log.Infof("Deleting vserver %v", v)
if afMap, ok := lb.vservers[v.Name]; !ok {
return fmt.Errorf("deleting non-existent Vserver: %v", v)
} else if _, ok := afMap[af]; !ok {
return fmt.Errorf("deleting wrong AF for Vserver %v: %v", v, af)
} else {
delete(afMap, af)
if len(afMap) == 0 {
delete(lb.vservers, v.Name)
}
}
return nil
}

func newTestEngine() *Engine {
e := NewEngine(nil)
e.ncc = &dummyNCC{}
e.lbInterface = &dummyLBInterface{}
e.lbInterface = newDummyLBInterface()
return e
}

Expand Down
33 changes: 33 additions & 0 deletions engine/testdata/re-ip/config_1.pb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
seesaw_vip <
fqdn: "seesaw-vip1.example.com."
ipv4: "192.168.36.16/26"
status: PRODUCTION
>
vserver <
name: "dns.resolver@au-syd"
rp: "foo"
entry_address <
fqdn: "dns-vip1.example.com."
ipv4: "192.168.36.1/24"
status: PRODUCTION
>
vserver_entry <
protocol: UDP
port: 53
persistence: 100
healthcheck <
type: HTTP
send: "foo"
receive: "bar"
code: 200
mode: DSR
>
>
backend: <
host: <
fqdn: "dns1-1.example.com."
ipv4: "192.168.37.2/26"
status: PRODUCTION
>
>
>
33 changes: 33 additions & 0 deletions engine/testdata/re-ip/config_2.pb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
seesaw_vip <
fqdn: "seesaw-vip1.example.com."
ipv4: "192.168.36.16/26"
status: PRODUCTION
>
vserver <
name: "dns.resolver@au-syd"
rp: "foo"
entry_address <
fqdn: "dns-vip1.example.com."
ipv4: "192.168.36.5/24"
status: PRODUCTION
>
vserver_entry <
protocol: UDP
port: 53
persistence: 100
healthcheck <
type: HTTP
send: "foo"
receive: "bar"
code: 200
mode: DSR
>
>
backend: <
host: <
fqdn: "dns1-1.example.com."
ipv4: "192.168.37.2/26"
status: PRODUCTION
>
>
>
33 changes: 33 additions & 0 deletions engine/testdata/re-ip/config_3.pb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
seesaw_vip <
fqdn: "seesaw-vip1.example.com."
ipv4: "192.168.36.16/26"
status: PRODUCTION
>
vserver <
name: "dns.resolver@au-syd"
rp: "foo"
entry_address <
fqdn: "dns-vip1.example.com."
ipv4: "192.168.36.5/24"
status: DISABLED
>
vserver_entry <
protocol: UDP
port: 53
persistence: 100
healthcheck <
type: HTTP
send: "foo"
receive: "bar"
code: 200
mode: DSR
>
>
backend: <
host: <
fqdn: "dns1-1.example.com."
ipv4: "192.168.37.2/26"
status: DISABLED
>
>
>
33 changes: 33 additions & 0 deletions engine/testdata/re-ip/config_4.pb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
seesaw_vip <
fqdn: "seesaw-vip1.example.com."
ipv4: "192.168.36.16/26"
status: PRODUCTION
>
vserver <
name: "dns.resolver@au-syd"
rp: "foo"
entry_address <
fqdn: "dns-anycast.example.com."
ipv4: "192.168.255.1/24"
status: PRODUCTION
>
vserver_entry <
protocol: UDP
port: 53
persistence: 100
healthcheck <
type: HTTP
send: "foo"
receive: "bar"
code: 200
mode: DSR
>
>
backend: <
host: <
fqdn: "dns1-1.example.com."
ipv4: "192.168.37.2/26"
status: PRODUCTION
>
>
>
33 changes: 33 additions & 0 deletions engine/testdata/re-ip/config_5.pb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
seesaw_vip <
fqdn: "seesaw-vip1.example.com."
ipv4: "192.168.36.16/26"
status: PRODUCTION
>
vserver <
name: "dns.resolver@au-syd"
rp: "foo"
entry_address <
fqdn: "dns-anycast.example.com."
ipv4: "192.168.255.1/24"
status: DISABLED
>
vserver_entry <
protocol: UDP
port: 53
persistence: 100
healthcheck <
type: HTTP
send: "foo"
receive: "bar"
code: 200
mode: DSR
>
>
backend: <
host: <
fqdn: "dns1-1.example.com."
ipv4: "192.168.37.2/26"
status: DISABLED
>
>
>
23 changes: 20 additions & 3 deletions engine/vserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ type vserver struct {
services map[serviceKey]*service
checks map[checkKey]*check
active map[seesaw.IP]bool
lbVservers map[seesaw.IP]*seesaw.Vserver
vips map[seesaw.VIP]bool
lbVservers map[seesaw.IP]*seesaw.Vserver // vservers with configured iptables rules
vips map[seesaw.VIP]bool // unicast VIPs

vserverOverride seesaw.VserverOverride
overrideChan chan seesaw.Override
Expand Down Expand Up @@ -675,6 +675,19 @@ func (v *vserver) configUpdate() {
}
}

// If a VIP has been re-IP'd or has no services configured, remove the old
// VIP from the interface.
needVIPs := make(map[seesaw.IP]bool)
for _, svc := range v.services {
needVIPs[svc.ip] = true
}
for vip := range v.vips {
if !needVIPs[vip.IP] {
log.Infof("%v: unconfiguring no longer needed VIP %v", v, vip.IP)
v.unconfigureVIP(&vip)
}
}

checks := v.expandChecks()
for k, oldCheck := range v.checks {
if checks[k] != nil {
Expand All @@ -683,6 +696,7 @@ func (v *vserver) configUpdate() {
}
}
v.checks = checks
// TODO(baptr): Should this only happen if it's enabled?
v.configureVIPs()
return
}
Expand All @@ -695,6 +709,8 @@ func (v *vserver) deleteService(s *service) {
}
log.Infof("%v: deleting service: %v", v, s)
delete(v.services, s.serviceKey)
// TODO(baptr): Once service contains seesaw.VIP, move check and
// unconfigureVIP here.
}

// handleCheckNotification processes a checkNotification, bringing
Expand Down Expand Up @@ -1284,6 +1300,7 @@ func (v *vserver) down(ip seesaw.IP) {
if err := v.engine.lbInterface.DeleteVserver(v.lbVservers[ip], ip.AF()); err != nil {
log.Fatalf("%v: failed to delete Vserver: %v", v, err)
}
delete(v.lbVservers, ip)
}
// TODO(jsing): Should we delay while the BGP routes propagate?

Expand Down Expand Up @@ -1354,7 +1371,7 @@ func (v *vserver) unconfigureVIP(vip *seesaw.VIP) {
if err := v.engine.lbInterface.DeleteVserver(v.lbVservers[vip.IP], vip.IP.AF()); err != nil {
log.Fatalf("%v: failed to delete Vserver: %v", v, err)
}
// TODO(baptr): delete(v.lbVservers, vip.IP)?
delete(v.lbVservers, vip.IP)
}
delete(v.vips, *vip)
}
Expand Down
Loading

0 comments on commit de11b28

Please sign in to comment.