[Feature] More configability for ingresses

ray-operator/apis/config/v1alpha1/configuration_types.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
@@ -73,6 +74,17 @@ type Configuration struct {
 
 	// EnableMetrics indicates whether KubeRay operator should emit control plane metrics.
 	EnableMetrics bool `json:"enableMetrics,omitempty"`
+
+	// Host used for Ray Dashboard ingresses. The host will be the same for all `RayClusters` and they
+	// will be differentiated by their paths.
+	IngressHost        string                    `json:"ingressHost,omitempty"`
+
+	// TLS configuration for the Ray Dashboard ingresses. Applies to all `RayClusters`.
+	IngressTLS         []networkingv1.IngressTLS `json:"ingressTLS,omitempty"`
+
+	// Default annotations for the Ray Dashboard ingresses. Annotations on the `RayCluster` will override
+	// these on a case-by-case basis.
+	IngressAnnotations map[string]string         `json:"ingressAnnotations,omitempty"`
 }
 
 func (config Configuration) GetDashboardClient(mgr manager.Manager) func() utils.RayDashboardClientInterface {

ray-operator/controllers/ray/common/ingress.go

@@ -15,7 +15,7 @@ const IngressClassAnnotationKey = "kubernetes.io/ingress.class"
 
 // BuildIngressForHeadService Builds the ingress for head service dashboard.
 // This is used to expose dashboard for external traffic.
-func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) (*networkingv1.Ingress, error) {
+func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster, host string, ingressTLS []networkingv1.IngressTLS, defaultAnnotations map[string]string) (*networkingv1.Ingress, error) {
 	log := ctrl.LoggerFrom(ctx)
 
 	labels := map[string]string{
@@ -32,10 +32,16 @@ func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) (
 	excludeSet := map[string]struct{}{
 		IngressClassAnnotationKey: {},
 	}
-	annotation := map[string]string{}
+	annotations := map[string]string{}
+	for key, value := range defaultAnnotations {
+		if _, ok := excludeSet[key]; !ok {
+			annotations[key] = value
+		}
+	}
+	// cluster.Annotations takes precedence, so we add these after defaultAnnotations.
 	for key, value := range cluster.Annotations {
 		if _, ok := excludeSet[key]; !ok {
-			annotation[key] = value
+			annotations[key] = value
 		}
 	}
 
@@ -70,11 +76,13 @@ func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) (
 			Name:        utils.GenerateIngressName(cluster.Name),
 			Namespace:   cluster.Namespace,
 			Labels:      labels,
-			Annotations: annotation,
+			Annotations: annotations,
 		},
 		Spec: networkingv1.IngressSpec{
+			TLS: ingressTLS,
 			Rules: []networkingv1.IngressRule{
 				{
+					Host: host,
 					IngressRuleValue: networkingv1.IngressRuleValue{
 						HTTP: &networkingv1.HTTPIngressRuleValue{
 							Paths: paths,
@@ -85,8 +93,13 @@ func BuildIngressForHeadService(ctx context.Context, cluster rayv1.RayCluster) (
 		},
 	}
 
-	// Get ingress class name from rayCluster annotations. this is a required field to use ingress.
-	if ingressClassName, ok := cluster.Annotations[IngressClassAnnotationKey]; !ok {
+	// First try to get ingress class name from rayCluster annotations.
+	ingressClassName, ok := cluster.Annotations[IngressClassAnnotationKey]
+	if !ok {
+		ingressClassName, ok = defaultAnnotations[IngressClassAnnotationKey]
+	}
+
+	if !ok {
 		log.Info("Ingress class annotation is not set for the cluster.", "clusterNamespace", cluster.Namespace, "clusterName", cluster.Name)
 	} else {
 		// TODO: in AWS EKS, set up IngressClassName will cause an error due to conflict with annotation.

ray-operator/controllers/ray/common/ingress_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
@@ -19,6 +20,8 @@ var instanceWithIngressEnabled = &rayv1.RayCluster{
 		Namespace: "default",
 		Annotations: map[string]string{
 			IngressClassAnnotationKey: "nginx",
+			"annotation0":             "value",
+			"annotation1":             "value",
 		},
 	},
 	Spec: rayv1.RayClusterSpec{
@@ -74,15 +77,18 @@ var instanceWithIngressEnabledWithoutIngressClass = &rayv1.RayCluster{
 
 // only throw warning message and rely on Kubernetes to assign default ingress class
 func TestBuildIngressForHeadServiceWithoutIngressClass(t *testing.T) {
-	ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabledWithoutIngressClass)
+	ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabledWithoutIngressClass, "", []networkingv1.IngressTLS{}, map[string]string{})
 	assert.NotNil(t, ingress)
 	require.NoError(t, err)
 }
 
 func TestBuildIngressForHeadService(t *testing.T) {
-	ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabled)
+	ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabled, "", []networkingv1.IngressTLS{}, map[string]string{})
 	require.NoError(t, err)
 
+	// annotations count
+	assert.Len(t, ingress.Annotations, 2)
+
 	// check ingress.class annotation
 	assert.Equal(t, instanceWithIngressEnabled.Name, ingress.Labels[utils.RayClusterLabelKey])
 
@@ -107,4 +113,45 @@ func TestBuildIngressForHeadService(t *testing.T) {
 	for _, path := range paths {
 		assert.Equal(t, headSvcName, path.Backend.Service.Name)
 	}
+
+	// check host
+	assert.Equal(t, ingress.Spec.Rules[0].Host, "")
+
+	// tls count
+	assert.Len(t, ingress.Spec.TLS, 0)
+}
+
+func TestBuildIngressForHeadServiceWithControllerConfigs(t *testing.T) {
+	host := "ray.example.com"
+	tls := []networkingv1.IngressTLS{
+		{
+			Hosts:      []string{host},
+			SecretName: "ray-tls-secret",
+		},
+	}
+	ingressClass := "different-ingress-class"
+	annotations := map[string]string{"annotation0": "value2", "annotation1": "value2", IngressClassAnnotationKey: ingressClass}
+	ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabledWithoutIngressClass, host, tls, annotations)
+	require.NoError(t, err)
+
+	assert.Equal(t, *ingress.Spec.IngressClassName, ingressClass)
+	assert.Equal(t, ingress.Annotations, map[string]string{
+		"annotation0": "value2", "annotation1": "value2",
+	})
+	assert.Equal(t, ingress.Spec.Rules[0].Host, host)
+	assert.Equal(t, ingress.Spec.TLS, tls)
+}
+
+func TestBuildIngressForHeadServiceClusterSpecificAnnotationsTakePrecedence(t *testing.T) {
+	annotations := map[string]string{"annotation0": "value2", "annotation2": "value2", IngressClassAnnotationKey: "different-ingress-class"}
+	ingress, err := BuildIngressForHeadService(context.Background(), *instanceWithIngressEnabled, "", []networkingv1.IngressTLS{}, annotations)
+	require.NoError(t, err)
+
+	delete(annotations, IngressClassAnnotationKey)
+	assert.Equal(t, ingress.Annotations, map[string]string{
+		"annotation0": "value", // Overridden by cluster annotation
+		"annotation1": "value",
+		"annotation2": "value2",
+	})
+	assert.Equal(t, instanceWithIngressEnabled.Annotations[IngressClassAnnotationKey], *ingress.Spec.IngressClassName)
 }

ray-operator/controllers/ray/raycluster_controller.go

@@ -96,6 +96,9 @@ type RayClusterReconcilerOptions struct {
 	HeadSidecarContainers    []corev1.Container
 	WorkerSidecarContainers  []corev1.Container
 	IsOpenShift              bool
+	IngressHost              string
+	IngressTLS               []networkingv1.IngressTLS
+	IngressAnnotations       map[string]string
 }
 
 // Reconcile reads that state of the cluster for a RayCluster object and makes changes based on it
@@ -478,7 +481,7 @@ func (r *RayClusterReconciler) reconcileIngressKubernetes(ctx context.Context, i
 	}
 
 	if len(headIngresses.Items) == 0 {
-		ingress, err := common.BuildIngressForHeadService(ctx, *instance)
+		ingress, err := common.BuildIngressForHeadService(ctx, *instance, r.options.IngressHost, r.options.IngressTLS, r.options.IngressAnnotations)
 		if err != nil {
 			return err
 		}

ray-operator/main.go

@@ -251,6 +251,9 @@ func main() {
 		WorkerSidecarContainers:  config.WorkerSidecarContainers,
 		IsOpenShift:              utils.GetClusterType(),
 		RayClusterMetricsManager: rayClusterMetricsManager,
+		IngressHost:              config.IngressHost,
+		IngressTLS:               config.IngressTLS,
+		IngressAnnotations:       config.IngressAnnotations,
 	}
 	exitOnError(ray.NewReconciler(ctx, mgr, rayClusterOptions, config).SetupWithManager(mgr, config.ReconcileConcurrency),
 		"unable to create controller", "controller", "RayCluster")

ray-operator/main_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 
@@ -97,6 +98,42 @@ workerSidecarContainers:
 			},
 			expectErr: false,
 		},
+		{
+			name: "config with ingress options",
+			configData: `apiVersion: config.ray.io/v1alpha1
+kind: Configuration
+ingressHost: ray.example.com
+ingressTLS:
+- hosts:
+  - ray.example.com
+  secretName: ray-tls-secret
+ingressAnnotations:
+  annotation0: value0
+  annotation1: value1
+`,
+			expectedConfig: configapi.Configuration{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Configuration",
+					APIVersion: "config.ray.io/v1alpha1",
+				},
+				MetricsAddr:          ":8080",
+				ProbeAddr:            ":8082",
+				EnableLeaderElection: ptr.To(true),
+				ReconcileConcurrency: 1,
+				IngressHost:          "ray.example.com",
+				IngressTLS: []networkingv1.IngressTLS{
+					{
+						Hosts:      []string{"ray.example.com"},
+						SecretName: "ray-tls-secret",
+					},
+				},
+				IngressAnnotations: map[string]string{
+					"annotation0": "value0",
+					"annotation1": "value1",
+				},
+			},
+			expectErr: false,
+		},
 		{
 			name: "unknown filed ignored",
 			configData: `apiVersion: config.ray.io/v1alpha1