Skip to content

Commit

Permalink
Feature an experimental "localhost" mode for sidecars
Browse files Browse the repository at this point in the history 
- Adds a new flag so that xDS server can run in a dummy "localhost" mode
  where EDS responses contain a single local (127.0.0.1) endpoint where
  we expect a sidecar to be listening
- Adds New Linear cahces and implements DELTA_GRPC streams support to be
  able to work with envoy on demand feature
- Adds a Kyverno rule to be able to inject envoy sidecars and use
  on-demand discovery based on configuration served via the main xDS
  server.
- Normalise annotations for injections
  • Loading branch information
@ffilippopoulos
ffilippopoulos committed Jan 28, 2025
1 parent fa8af65 commit 136dc92
Show file tree
Hide file tree
Showing 17 changed files with 932 additions and 439 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.22-alpine AS build
FROM golang:1.23-alpine AS build
WORKDIR /go/src/github.com/utilitywarehouse/semaphore-xds
COPY . /go/src/github.com/utilitywarehouse/semaphore-xds
ENV CGO_ENABLED=0
Expand Down
32 changes: 16 additions & 16 deletions controller/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestReconcileServices_LabelledService(t *testing.T) {
"./test-resources/labelled_service.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -82,7 +82,7 @@ func TestReconcileServices_LabelledServiceLbPolicy(t *testing.T) {
"./test-resources/labelled_service_ring_hash_balancer.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -119,7 +119,7 @@ func TestReconcileServices_LabelledServiceInvalidLbPolicy(t *testing.T) {
"./test-resources/labelled_service_invalid_balancer.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -156,7 +156,7 @@ func TestReconcileServices_XdsService(t *testing.T) {
"./test-resources/xds_service.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -199,7 +199,7 @@ func TestReconcileServices_XdsServiceNotExistent(t *testing.T) {
"./test-resources/xds_service_not_existent.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -233,7 +233,7 @@ func TestReconcileServices_XdsServiceDelete(t *testing.T) {
"./test-resources/xds_service.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -284,7 +284,7 @@ func TestReconcileLocalEndpointSlice_SnapOnUpdate(t *testing.T) {
"./test-resources/xds_service.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -318,7 +318,7 @@ func TestReconcileLocalEndpointSlice_NotFound(t *testing.T) {
"./test-resources/endpointslice.yaml",
)
client.EndpointSliceApiError(kubeerror.NewNotFound(schema.GroupResource{Resource: "endpointslice"}, "foo"))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand All @@ -345,7 +345,7 @@ func TestReconcileLocalEndpointSlice_NonXdsService(t *testing.T) {
client := kube.NewClientMock(
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down Expand Up @@ -373,7 +373,7 @@ func TestReconcileServices_XdsServiceWithRemoteEndpoints(t *testing.T) {
remoteClient := kube.NewClientMock(
"./test-resources/endpointslice-remote.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
localClient,
[]kube.Client{remoteClient},
Expand Down Expand Up @@ -436,7 +436,7 @@ func TestReconcileServices_XdsServiceWithRemoteEndpoints_NoRemoteEndpoints(t *te
remoteClient := kube.NewClientMock(
"./test-resources/endpointslice-remote.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
localClient,
[]kube.Client{remoteClient},
Expand Down Expand Up @@ -487,7 +487,7 @@ func TestReconcileServices_XdsServiceWithOnlyRemoteEndpoints(t *testing.T) {
remoteClient := kube.NewClientMock(
"./test-resources/endpointslice-remote.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
localClient,
[]kube.Client{remoteClient},
Expand Down Expand Up @@ -539,7 +539,7 @@ func TestReconcileServices_XdsServiceWithRemoteEndpointsAndLocalPriority(t *test
remoteClient := kube.NewClientMock(
"./test-resources/endpointslice-remote.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
localClient,
[]kube.Client{remoteClient},
Expand Down Expand Up @@ -601,7 +601,7 @@ func TestReconcileServices_XdsServiceWithOnlyRemoteEndpointsAndLocalPriority(t *
remoteClient := kube.NewClientMock(
"./test-resources/endpointslice-remote.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
localClient,
[]kube.Client{remoteClient},
Expand Down Expand Up @@ -656,7 +656,7 @@ func TestReconcileLocalEndpointSlices_XdsServiceWithEmptyLocalEndpoints(t *testi
remoteClient := kube.NewClientMock(
"./test-resources/endpointslice-remote.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
localClient,
[]kube.Client{remoteClient},
Expand Down Expand Up @@ -700,7 +700,7 @@ func TestReconcileServices_XdsServiceWithRingHash(t *testing.T) {
"./test-resources/xds_service_ring_hash_balancing.yaml",
"./test-resources/endpointslice.yaml",
)
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0))
snapshotter := xds.NewSnapshotter("", testSnapshotterListenPort, float64(0), float64(0), false)
controller := NewController(
client,
[]kube.Client{},
Expand Down
1 change: 1 addition & 0 deletions deploy/kustomize/kyverno/mutate/kustomization.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- mutate-semaphore-xds-clients-env.yaml
- mutate-semaphore-xds-envoy-sidecar.yaml
191 changes: 100 additions & 91 deletions deploy/kustomize/kyverno/mutate/mutate-semaphore-xds-clients-env.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,102 +14,111 @@ spec:
background: false
mutateExistingOnPolicyUpdate: false
rules:
- name: xds-clients-inject-env
match:
resources:
kinds:
- Pod
operations:
- CREATE
selector:
matchLabels:
xds.semaphore.uw.systems/client: "true"
mutate:
patchStrategicMerge:
spec:
initContainers:
- (name): "*"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: GRPC_XDS_BOOTSTRAP_CONFIG
value: >-
{
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.local:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
],
"node":{
"id":"{{request.object.metadata.namespace}}/\$(POD_NAME)",
"locality":{}
},
"authorities": {
"aws": {
- name: xds-clients-inject-env
match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
selector:
matchLabels:
xds.semaphore.uw.systems/client: "true"
- resources:
kinds:
- Pod
operations:
- CREATE
selector:
matchLabels:
xds.semaphore.uw.systems/client: "native"
mutate:
patchStrategicMerge:
spec:
initContainers:
- (name): "*"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: GRPC_XDS_BOOTSTRAP_CONFIG
value: >-
{
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.aws:18000",
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.local:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"gcp": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.gcp:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"merit": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.merit:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
],
"node":{
"id":"{{request.object.metadata.namespace}}/\$(POD_NAME)",
"locality":{}
},
"authorities": {
"aws": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.aws:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"gcp": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.gcp:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"merit": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.merit:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
}
}
}
}
}
containers:
- (name): "*"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: GRPC_XDS_BOOTSTRAP_CONFIG
value: >-
{
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.local:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
],
"node":{
"id":"{{request.object.metadata.namespace}}/\$(POD_NAME)",
"locality":{}
},
"authorities": {
"aws": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.aws:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"gcp": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.gcp:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"merit": {
containers:
- (name): "*"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: GRPC_XDS_BOOTSTRAP_CONFIG
value: >-
{
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.merit:18000",
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.local:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
],
"node":{
"id":"{{request.object.metadata.namespace}}/\$(POD_NAME)",
"locality":{}
},
"authorities": {
"aws": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.aws:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"gcp": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.gcp:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
},
"merit": {
"xds_servers": [{
"server_uri": "semaphore-xds.sys-semaphore.svc.cluster.merit:18000",
"channel_creds": [{"type": "insecure"}],
"server_features": ["xds_v3"]}
]
}
}
}
}
}
Loading

0 comments on commit 136dc92

Please sign in to comment.