diff --git a/docs/reference/api.md b/docs/reference/api.md index aef1aef4235..e966d6bbf9c 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -28,10 +28,6 @@ AuthMode describes the authentication mode for the Ray cluster. _Appears in:_ - [AuthOptions](#authoptions) -| Field | Description | -| --- | --- | -| `disabled` | AuthModeDisabled disables authentication.
| -| `token` | AuthModeToken enables token-based authentication.
| #### AuthOptions @@ -87,10 +83,6 @@ _Validation:_ _Appears in:_ - [AutoscalerOptions](#autoscaleroptions) -| Field | Description | -| --- | --- | -| `v1` | | -| `v2` | | #### ClusterUpgradeOptions @@ -160,12 +152,6 @@ _Appears in:_ - [DeletionPolicy](#deletionpolicy) - [DeletionRule](#deletionrule) -| Field | Description | -| --- | --- | -| `DeleteCluster` | | -| `DeleteWorkers` | | -| `DeleteSelf` | | -| `DeleteNone` | | #### DeletionRule @@ -193,15 +179,18 @@ _Appears in:_ DeletionStrategy configures automated cleanup after the RayJob reaches a terminal state. Two mutually exclusive styles are supported: + Legacy: provide both onSuccess and onFailure (deprecated; removal planned for 1.6.0). May be combined with shutdownAfterJobFinishes and (optionally) global TTLSecondsAfterFinished. Rules: provide deletionRules (non-empty list). Rules mode is incompatible with shutdownAfterJobFinishes, legacy fields, and the global TTLSecondsAfterFinished (use per‑rule condition.ttlSeconds instead). + Semantics: - A non-empty deletionRules selects rules mode; empty lists are treated as unset. - Legacy requires both onSuccess and onFailure; specifying only one is invalid. - Global TTLSecondsAfterFinished > 0 requires shutdownAfterJobFinishes=true; therefore it cannot be used with rules mode or with legacy alone (no shutdown). - Feature gate RayJobDeletionPolicy must be enabled when this block is present. + Validation: - CRD XValidations prevent mixing legacy fields with deletionRules and enforce legacy completeness. - Controller logic enforces rules vs shutdown exclusivity and TTL constraints. @@ -275,12 +264,6 @@ _Underlying type:_ _string_ _Appears in:_ - [RayJobSpec](#rayjobspec) -| Field | Description | -| --- | --- | -| `K8sJobMode` | | -| `HTTPMode` | | -| `InteractiveMode` | | -| `SidecarMode` | | #### RayCluster @@ -359,10 +342,6 @@ _Validation:_ _Appears in:_ - [RayClusterUpgradeStrategy](#rayclusterupgradestrategy) -| Field | Description | -| --- | --- | -| `Recreate` | During upgrade, Recreate strategy will delete all existing pods before creating new ones
| -| `None` | No new pod will be created while the strategy is set to None
| #### RayCronJob @@ -533,11 +512,6 @@ _Underlying type:_ _string_ _Appears in:_ - [RayServiceUpgradeStrategy](#rayserviceupgradestrategy) -| Field | Description | -| --- | --- | -| `NewClusterWithIncrementalUpgrade` | During upgrade, NewClusterWithIncrementalUpgrade strategy will create an upgraded cluster to gradually scale
and migrate traffic to using Gateway API.
| -| `NewCluster` | During upgrade, NewCluster strategy will create new upgraded cluster and switch to it when it becomes ready
| -| `None` | No new cluster will be created while the strategy is set to None
| #### RedisCredential diff --git a/go.mod b/go.mod index a98afa87b7c..5d82fb4d447 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/ray-project/kuberay/ray-operator v0.0.0 github.com/rs/cors v1.11.1 github.com/rs/zerolog v1.34.0 - github.com/spf13/cobra v1.9.1 + github.com/spf13/cobra v1.10.0 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 @@ -28,10 +28,10 @@ require ( google.golang.org/grpc v1.75.1 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.34.3 - k8s.io/apimachinery v0.34.3 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 k8s.io/cli-runtime v0.34.1 - k8s.io/client-go v0.34.3 + k8s.io/client-go v0.35.0 k8s.io/klog/v2 v2.130.1 k8s.io/kubectl v0.34.1 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 @@ -121,8 +121,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.34.3 // indirect - k8s.io/apiserver v0.34.3 // indirect - k8s.io/component-base v0.34.3 // indirect + k8s.io/apiserver v0.35.0 // indirect + k8s.io/component-base v0.35.0 // indirect k8s.io/component-helpers v0.34.1 // indirect k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect sigs.k8s.io/gateway-api v1.4.1 // indirect diff --git a/go.sum b/go.sum index ea726f2b77a..26586b81381 100644 --- a/go.sum +++ b/go.sum @@ -925,9 +925,9 @@ github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY52 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= +github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= @@ -2142,20 +2142,20 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.34.3 h1:D12sTP257/jSH2vHV2EDYrb16bS7ULlHpdNdNhEw2S4= -k8s.io/api v0.34.3/go.mod h1:PyVQBF886Q5RSQZOim7DybQjAbVs8g7gwJNhGtY5MBk= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= k8s.io/apiextensions-apiserver v0.34.3 h1:p10fGlkDY09eWKOTeUSioxwLukJnm+KuDZdrW71y40g= k8s.io/apiextensions-apiserver v0.34.3/go.mod h1:aujxvqGFRdb/cmXYfcRTeppN7S2XV/t7WMEc64zB5A0= -k8s.io/apimachinery v0.34.3 h1:/TB+SFEiQvN9HPldtlWOTp0hWbJ+fjU+wkxysf/aQnE= -k8s.io/apimachinery v0.34.3/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/apiserver v0.34.3 h1:uGH1qpDvSiYG4HVFqc6A3L4CKiX+aBWDrrsxHYK0Bdo= -k8s.io/apiserver v0.34.3/go.mod h1:QPnnahMO5C2m3lm6fPW3+JmyQbvHZQ8uudAu/493P2w= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/apiserver v0.35.0 h1:CUGo5o+7hW9GcAEF3x3usT3fX4f9r8xmgQeCBDaOgX4= +k8s.io/apiserver v0.35.0/go.mod h1:QUy1U4+PrzbJaM3XGu2tQ7U9A4udRRo5cyxkFX0GEds= k8s.io/cli-runtime v0.34.1 h1:btlgAgTrYd4sk8vJTRG6zVtqBKt9ZMDeQZo2PIzbL7M= k8s.io/cli-runtime v0.34.1/go.mod h1:aVA65c+f0MZiMUPbseU/M9l1Wo2byeaGwUuQEQVVveE= -k8s.io/client-go v0.34.3 h1:wtYtpzy/OPNYf7WyNBTj3iUA0XaBHVqhv4Iv3tbrF5A= -k8s.io/client-go v0.34.3/go.mod h1:OxxeYagaP9Kdf78UrKLa3YZixMCfP6bgPwPwNBQBzpM= -k8s.io/component-base v0.34.3 h1:zsEgw6ELqK0XncCQomgO9DpUIzlrYuZYA0Cgo+JWpVk= -k8s.io/component-base v0.34.3/go.mod h1:5iIlD8wPfWE/xSHTRfbjuvUul2WZbI2nOUK65XL0E/c= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= +k8s.io/component-base v0.35.0 h1:+yBrOhzri2S1BVqyVSvcM3PtPyx5GUxCK2tinZz1G94= +k8s.io/component-base v0.35.0/go.mod h1:85SCX4UCa6SCFt6p3IKAPej7jSnF3L8EbfSyMZayJR0= k8s.io/component-helpers v0.34.1 h1:gWhH3CCdwAx5P3oJqZKb4Lg5FYZTWVbdWtOI8n9U4XY= k8s.io/component-helpers v0.34.1/go.mod h1:4VgnUH7UA/shuBur+OWoQC0xfb69sy/93ss0ybZqm3c= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml index 575408824d9..450310fedba 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml @@ -4315,6 +4315,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4509,6 +4513,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8196,6 +8212,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8390,6 +8410,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -12636,6 +12668,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -12830,6 +12866,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -16485,6 +16533,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -16679,6 +16731,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml index a24c57a20ae..23cdb163f79 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml @@ -4400,6 +4400,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4594,6 +4598,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8281,6 +8297,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8475,6 +8495,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -12113,6 +12145,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -12307,6 +12343,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml index 2444881c176..f4e0260e101 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml @@ -4406,6 +4406,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4600,6 +4604,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8287,6 +8303,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8481,6 +8501,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -12119,6 +12151,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -12313,6 +12349,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -16585,6 +16633,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -16779,6 +16831,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -20434,6 +20498,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -20628,6 +20696,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -24258,6 +24338,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -24452,6 +24536,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml index c99e2b253bf..14a20c4b71b 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml @@ -4295,6 +4295,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4489,6 +4493,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8176,6 +8192,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8370,6 +8390,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -13047,6 +13079,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -13241,6 +13277,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -16896,6 +16944,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -17090,6 +17142,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml index 575408824d9..450310fedba 100644 --- a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml @@ -4315,6 +4315,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4509,6 +4513,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8196,6 +8212,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8390,6 +8410,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -12636,6 +12668,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -12830,6 +12866,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -16485,6 +16533,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -16679,6 +16731,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml b/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml index a24c57a20ae..23cdb163f79 100644 --- a/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml +++ b/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml @@ -4400,6 +4400,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4594,6 +4598,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8281,6 +8297,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8475,6 +8495,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -12113,6 +12145,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -12307,6 +12343,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml index 2444881c176..f4e0260e101 100644 --- a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml @@ -4406,6 +4406,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4600,6 +4604,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8287,6 +8303,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8481,6 +8501,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -12119,6 +12151,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -12313,6 +12349,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -16585,6 +16633,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -16779,6 +16831,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -20434,6 +20498,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -20628,6 +20696,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -24258,6 +24338,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -24452,6 +24536,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/ray-operator/config/crd/bases/ray.io_rayservices.yaml b/ray-operator/config/crd/bases/ray.io_rayservices.yaml index c99e2b253bf..14a20c4b71b 100644 --- a/ray-operator/config/crd/bases/ray.io_rayservices.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayservices.yaml @@ -4295,6 +4295,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -4489,6 +4493,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -8176,6 +8192,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -8370,6 +8390,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -13047,6 +13079,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -13241,6 +13277,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -16896,6 +16944,10 @@ spec: type: integer signerName: type: string + userAnnotations: + additionalProperties: + type: string + type: object required: - keyType - signerName @@ -17090,6 +17142,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + properties: + name: + type: string + podGroup: + type: string + podGroupReplicaKey: + type: string + required: + - name + - podGroup + type: object required: - containers type: object diff --git a/ray-operator/go.mod b/ray-operator/go.mod index 8e0a3e6e96f..db08db5eeec 100644 --- a/ray-operator/go.mod +++ b/ray-operator/go.mod @@ -19,12 +19,12 @@ require ( go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.34.3 - k8s.io/apimachinery v0.34.3 - k8s.io/apiserver v0.34.3 - k8s.io/client-go v0.34.3 - k8s.io/code-generator v0.34.3 - k8s.io/component-base v0.34.3 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/apiserver v0.35.0 + k8s.io/client-go v0.35.0 + k8s.io/code-generator v0.35.0 + k8s.io/component-base v0.35.0 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 sigs.k8s.io/controller-runtime v0.22.4 @@ -76,7 +76,6 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.4 // indirect github.com/prometheus/procfs v0.19.2 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/otel v1.39.0 // indirect diff --git a/ray-operator/go.sum b/ray-operator/go.sum index f04b159a4aa..3a28ca82bd3 100644 --- a/ray-operator/go.sum +++ b/ray-operator/go.sum @@ -246,20 +246,20 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.3 h1:D12sTP257/jSH2vHV2EDYrb16bS7ULlHpdNdNhEw2S4= -k8s.io/api v0.34.3/go.mod h1:PyVQBF886Q5RSQZOim7DybQjAbVs8g7gwJNhGtY5MBk= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= k8s.io/apiextensions-apiserver v0.34.3 h1:p10fGlkDY09eWKOTeUSioxwLukJnm+KuDZdrW71y40g= k8s.io/apiextensions-apiserver v0.34.3/go.mod h1:aujxvqGFRdb/cmXYfcRTeppN7S2XV/t7WMEc64zB5A0= -k8s.io/apimachinery v0.34.3 h1:/TB+SFEiQvN9HPldtlWOTp0hWbJ+fjU+wkxysf/aQnE= -k8s.io/apimachinery v0.34.3/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/apiserver v0.34.3 h1:uGH1qpDvSiYG4HVFqc6A3L4CKiX+aBWDrrsxHYK0Bdo= -k8s.io/apiserver v0.34.3/go.mod h1:QPnnahMO5C2m3lm6fPW3+JmyQbvHZQ8uudAu/493P2w= -k8s.io/client-go v0.34.3 h1:wtYtpzy/OPNYf7WyNBTj3iUA0XaBHVqhv4Iv3tbrF5A= -k8s.io/client-go v0.34.3/go.mod h1:OxxeYagaP9Kdf78UrKLa3YZixMCfP6bgPwPwNBQBzpM= -k8s.io/code-generator v0.34.3 h1:6ipJKsJZZ9q21BO8I2jEj4OLN3y8/1n4aihKN0xKmQk= -k8s.io/code-generator v0.34.3/go.mod h1:oW73UPYpGLsbRN8Ozkhd6ZzkF8hzFCiYmvEuWZDroI4= -k8s.io/component-base v0.34.3 h1:zsEgw6ELqK0XncCQomgO9DpUIzlrYuZYA0Cgo+JWpVk= -k8s.io/component-base v0.34.3/go.mod h1:5iIlD8wPfWE/xSHTRfbjuvUul2WZbI2nOUK65XL0E/c= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/apiserver v0.35.0 h1:CUGo5o+7hW9GcAEF3x3usT3fX4f9r8xmgQeCBDaOgX4= +k8s.io/apiserver v0.35.0/go.mod h1:QUy1U4+PrzbJaM3XGu2tQ7U9A4udRRo5cyxkFX0GEds= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= +k8s.io/code-generator v0.35.0 h1:TvrtfKYZTm9oDF2z+veFKSCcgZE3Igv0svY+ehCmjHQ= +k8s.io/code-generator v0.35.0/go.mod h1:iS1gvVf3c/T71N5DOGYO+Gt3PdJ6B9LYSvIyQ4FHzgc= +k8s.io/component-base v0.35.0 h1:+yBrOhzri2S1BVqyVSvcM3PtPyx5GUxCK2tinZz1G94= +k8s.io/component-base v0.35.0/go.mod h1:85SCX4UCa6SCFt6p3IKAPej7jSnF3L8EbfSyMZayJR0= k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/authoptions.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/authoptions.go index 6f2cdaf680b..a79d01a8365 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/authoptions.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/authoptions.go @@ -8,7 +8,12 @@ import ( // AuthOptionsApplyConfiguration represents a declarative configuration of the AuthOptions type for use // with apply. +// +// AuthOptions defines the authentication options for a RayCluster. type AuthOptionsApplyConfiguration struct { + // Mode specifies the authentication mode. + // Supported values are "disabled" and "token". + // Defaults to "token". Mode *rayv1.AuthMode `json:"mode,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/autoscaleroptions.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/autoscaleroptions.go index aba56ccfd45..fa86b33188c 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/autoscaleroptions.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/autoscaleroptions.go @@ -9,17 +9,41 @@ import ( // AutoscalerOptionsApplyConfiguration represents a declarative configuration of the AutoscalerOptions type for use // with apply. +// +// AutoscalerOptions specifies optional configuration for the Ray autoscaler. type AutoscalerOptionsApplyConfiguration struct { - Resources *corev1.ResourceRequirements `json:"resources,omitempty"` - Image *string `json:"image,omitempty"` - ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` - IdleTimeoutSeconds *int32 `json:"idleTimeoutSeconds,omitempty"` - UpscalingMode *rayv1.UpscalingMode `json:"upscalingMode,omitempty"` - Version *rayv1.AutoscalerVersion `json:"version,omitempty"` - Env []corev1.EnvVar `json:"env,omitempty"` - EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"` - VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` + // Resources specifies optional resource request and limit overrides for the autoscaler container. + // Default values: 500m CPU request and limit. 512Mi memory request and limit. + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + // Image optionally overrides the autoscaler's container image. This override is provided for autoscaler testing and development. + Image *string `json:"image,omitempty"` + // ImagePullPolicy optionally overrides the autoscaler container's image pull policy. This override is provided for autoscaler testing and development. + ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + // SecurityContext defines the security options the container should be run with. + // If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + // More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + // IdleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. + // Defaults to 60 (one minute). It is not read by the KubeRay operator but by the Ray autoscaler. + IdleTimeoutSeconds *int32 `json:"idleTimeoutSeconds,omitempty"` + // UpscalingMode is "Conservative", "Default", or "Aggressive." + // Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. + // Default: Upscaling is not rate-limited. + // Aggressive: An alias for Default; upscaling is not rate-limited. + // It is not read by the KubeRay operator but by the Ray autoscaler. + UpscalingMode *rayv1.UpscalingMode `json:"upscalingMode,omitempty"` + // Version is the version of the Ray autoscaler. + // Setting this to v1 will explicitly use autoscaler v1. + // Setting this to v2 will explicitly use autoscaler v2. + // If this isn't set, the Ray version determines the autoscaler version. + // In Ray 2.47.0 and later, the default autoscaler version is v2. It's v1 before that. + Version *rayv1.AutoscalerVersion `json:"version,omitempty"` + // Optional list of environment variables to set in the autoscaler container. + Env []corev1.EnvVar `json:"env,omitempty"` + // Optional list of sources to populate environment variables in the autoscaler container. + EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"` + // Optional list of volumeMounts. This is needed for enabling TLS for the autoscaler container. + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` } // AutoscalerOptionsApplyConfiguration constructs a declarative configuration of the AutoscalerOptions type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/clusterupgradeoptions.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/clusterupgradeoptions.go index 1e43d339716..caa59bf0d76 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/clusterupgradeoptions.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/clusterupgradeoptions.go @@ -4,10 +4,17 @@ package v1 // ClusterUpgradeOptionsApplyConfiguration represents a declarative configuration of the ClusterUpgradeOptions type for use // with apply. +// +// These options are currently only supported for the IncrementalUpgrade type. type ClusterUpgradeOptionsApplyConfiguration struct { - MaxSurgePercent *int32 `json:"maxSurgePercent,omitempty"` - StepSizePercent *int32 `json:"stepSizePercent,omitempty"` - IntervalSeconds *int32 `json:"intervalSeconds,omitempty"` + // The capacity of serve requests the upgraded cluster should scale to handle each interval. + // Defaults to 100%. + MaxSurgePercent *int32 `json:"maxSurgePercent,omitempty"` + // The percentage of traffic to switch to the upgraded RayCluster at a set interval after scaling by MaxSurgePercent. + StepSizePercent *int32 `json:"stepSizePercent,omitempty"` + // The interval in seconds between transferring StepSize traffic from the old to new RayCluster. + IntervalSeconds *int32 `json:"intervalSeconds,omitempty"` + // The name of the Gateway Class installed by the Kubernetes Cluster admin. GatewayClassName *string `json:"gatewayClassName,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletioncondition.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletioncondition.go index ee816ecf5bf..0bcf6a6bc8b 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletioncondition.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletioncondition.go @@ -8,10 +8,22 @@ import ( // DeletionConditionApplyConfiguration represents a declarative configuration of the DeletionCondition type for use // with apply. +// +// DeletionCondition specifies the trigger conditions for a deletion action. +// Exactly one of JobStatus or JobDeploymentStatus must be specified: +// - JobStatus (application-level): Match the Ray job execution status. +// - JobDeploymentStatus (infrastructure-level): Match the RayJob deployment lifecycle status. This is particularly useful for cleaning up resources when Ray jobs fail to be submitted. type DeletionConditionApplyConfiguration struct { - JobStatus *rayv1.JobStatus `json:"jobStatus,omitempty"` + // JobStatus is the terminal status of the RayJob that triggers this condition. + // For the initial implementation, only "SUCCEEDED" and "FAILED" are supported. + JobStatus *rayv1.JobStatus `json:"jobStatus,omitempty"` + // JobDeploymentStatus is the terminal status of the RayJob deployment that triggers this condition. + // For the initial implementation, only "Failed" is supported. JobDeploymentStatus *rayv1.JobDeploymentStatus `json:"jobDeploymentStatus,omitempty"` - TTLSeconds *int32 `json:"ttlSeconds,omitempty"` + // TTLSeconds is the time in seconds from when the JobStatus or JobDeploymentStatus + // reaches the specified terminal state to when this deletion action should be triggered. + // The value must be a non-negative integer. + TTLSeconds *int32 `json:"ttlSeconds,omitempty"` } // DeletionConditionApplyConfiguration constructs a declarative configuration of the DeletionCondition type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionpolicy.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionpolicy.go index 1bb169cef00..ecbc699694a 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionpolicy.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionpolicy.go @@ -8,7 +8,13 @@ import ( // DeletionPolicyApplyConfiguration represents a declarative configuration of the DeletionPolicy type for use // with apply. +// +// DeletionPolicy is the legacy single-stage deletion policy. +// Deprecated: This struct is part of the legacy API. Use DeletionRule for new configurations. type DeletionPolicyApplyConfiguration struct { + // Policy is the action to take when the condition is met. + // This field is logically required when using the legacy OnSuccess/OnFailure policies. + // It is marked as '+optional' at the API level to allow the 'deletionRules' field to be used instead. Policy *rayv1.DeletionPolicyType `json:"policy,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionrule.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionrule.go index 91e4b50de99..939f9b113a1 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionrule.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionrule.go @@ -8,8 +8,13 @@ import ( // DeletionRuleApplyConfiguration represents a declarative configuration of the DeletionRule type for use // with apply. +// +// DeletionRule defines a single deletion action and its trigger condition. +// This is the new, recommended way to define deletion behavior. type DeletionRuleApplyConfiguration struct { - Policy *rayv1.DeletionPolicyType `json:"policy,omitempty"` + // Policy is the action to take when the condition is met. This field is required. + Policy *rayv1.DeletionPolicyType `json:"policy,omitempty"` + // The condition under which this deletion rule is triggered. This field is required. Condition *DeletionConditionApplyConfiguration `json:"condition,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionstrategy.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionstrategy.go index 034cce827cb..03195d69de2 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionstrategy.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/deletionstrategy.go @@ -4,10 +4,36 @@ package v1 // DeletionStrategyApplyConfiguration represents a declarative configuration of the DeletionStrategy type for use // with apply. +// +// DeletionStrategy configures automated cleanup after the RayJob reaches a terminal state. +// Two mutually exclusive styles are supported: +// +// Legacy: provide both onSuccess and onFailure (deprecated; removal planned for 1.6.0). May be combined with shutdownAfterJobFinishes and (optionally) global TTLSecondsAfterFinished. +// Rules: provide deletionRules (non-empty list). Rules mode is incompatible with shutdownAfterJobFinishes, legacy fields, and the global TTLSecondsAfterFinished (use per‑rule condition.ttlSeconds instead). +// +// Semantics: +// - A non-empty deletionRules selects rules mode; empty lists are treated as unset. +// - Legacy requires both onSuccess and onFailure; specifying only one is invalid. +// - Global TTLSecondsAfterFinished > 0 requires shutdownAfterJobFinishes=true; therefore it cannot be used with rules mode or with legacy alone (no shutdown). +// - Feature gate RayJobDeletionPolicy must be enabled when this block is present. +// +// Validation: +// - CRD XValidations prevent mixing legacy fields with deletionRules and enforce legacy completeness. +// - Controller logic enforces rules vs shutdown exclusivity and TTL constraints. +// - onSuccess/onFailure are deprecated; migration to deletionRules is encouraged. type DeletionStrategyApplyConfiguration struct { - OnSuccess *DeletionPolicyApplyConfiguration `json:"onSuccess,omitempty"` - OnFailure *DeletionPolicyApplyConfiguration `json:"onFailure,omitempty"` - DeletionRules []DeletionRuleApplyConfiguration `json:"deletionRules,omitempty"` + // OnSuccess is the deletion policy for a successful RayJob. + // Deprecated: Use `deletionRules` instead for more flexible, multi-stage deletion strategies. + // This field will be removed in release 1.6.0. + OnSuccess *DeletionPolicyApplyConfiguration `json:"onSuccess,omitempty"` + // OnFailure is the deletion policy for a failed RayJob. + // Deprecated: Use `deletionRules` instead for more flexible, multi-stage deletion strategies. + // This field will be removed in release 1.6.0. + OnFailure *DeletionPolicyApplyConfiguration `json:"onFailure,omitempty"` + // DeletionRules is a list of deletion rules, processed based on their trigger conditions. + // While the rules can be used to define a sequence, if multiple rules are overdue (e.g., due to controller downtime), + // the most impactful rule (e.g., DeleteSelf) will be executed first to prioritize resource cleanup. + DeletionRules []DeletionRuleApplyConfiguration `json:"deletionRules,omitempty"` } // DeletionStrategyApplyConfiguration constructs a declarative configuration of the DeletionStrategy type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/gcsfaulttoleranceoptions.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/gcsfaulttoleranceoptions.go index 1ddc93df177..4623be4a13c 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/gcsfaulttoleranceoptions.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/gcsfaulttoleranceoptions.go @@ -4,6 +4,8 @@ package v1 // GcsFaultToleranceOptionsApplyConfiguration represents a declarative configuration of the GcsFaultToleranceOptions type for use // with apply. +// +// GcsFaultToleranceOptions contains configs for GCS FT type GcsFaultToleranceOptionsApplyConfiguration struct { RedisUsername *RedisCredentialApplyConfiguration `json:"redisUsername,omitempty"` RedisPassword *RedisCredentialApplyConfiguration `json:"redisPassword,omitempty"` diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/headgroupspec.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/headgroupspec.go index c5233d65732..eed841cf677 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/headgroupspec.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/headgroupspec.go @@ -9,14 +9,27 @@ import ( // HeadGroupSpecApplyConfiguration represents a declarative configuration of the HeadGroupSpec type for use // with apply. +// +// HeadGroupSpec are the spec for the head pod type HeadGroupSpecApplyConfiguration struct { - Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` - HeadService *apicorev1.Service `json:"headService,omitempty"` - EnableIngress *bool `json:"enableIngress,omitempty"` - Resources map[string]string `json:"resources,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - RayStartParams map[string]string `json:"rayStartParams,omitempty"` - ServiceType *apicorev1.ServiceType `json:"serviceType,omitempty"` + // Template is the exact pod template used in K8s deployments, statefulsets, etc. + Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` + // HeadService is the Kubernetes service of the head pod. + HeadService *apicorev1.Service `json:"headService,omitempty"` + // EnableIngress indicates whether operator should create ingress object for head service or not. + EnableIngress *bool `json:"enableIngress,omitempty"` + // Resources specifies the resource quantities for the head group. + // These values override the resources passed to `rayStartParams` for the group, but + // have no effect on the resources set at the K8s Pod container level. + Resources map[string]string `json:"resources,omitempty"` + // Labels specifies the Ray node labels for the head group. + // These labels will also be added to the Pods of this head group and override the `--labels` + // argument passed to `rayStartParams`. + Labels map[string]string `json:"labels,omitempty"` + // RayStartParams are the params of the start command: node-manager-port, object-store-memory, ... + RayStartParams map[string]string `json:"rayStartParams,omitempty"` + // ServiceType is Kubernetes service type of the head service. it will be used by the workers to connect to the head pod + ServiceType *apicorev1.ServiceType `json:"serviceType,omitempty"` } // HeadGroupSpecApplyConfiguration constructs a declarative configuration of the HeadGroupSpec type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/headinfo.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/headinfo.go index 6a1ca44cd61..8b75b1e6a4c 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/headinfo.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/headinfo.go @@ -4,6 +4,8 @@ package v1 // HeadInfoApplyConfiguration represents a declarative configuration of the HeadInfo type for use // with apply. +// +// HeadInfo gives info about head type HeadInfoApplyConfiguration struct { PodIP *string `json:"podIP,omitempty"` ServiceIP *string `json:"serviceIP,omitempty"` diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/raycluster.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/raycluster.go index f1b884dce1d..637ceb67d0b 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/raycluster.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/raycluster.go @@ -10,11 +10,15 @@ import ( // RayClusterApplyConfiguration represents a declarative configuration of the RayCluster type for use // with apply. +// +// RayCluster is the Schema for the RayClusters API type RayClusterApplyConfiguration struct { + // Standard object metadata. metav1.TypeMetaApplyConfiguration `json:",inline"` *metav1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *RayClusterSpecApplyConfiguration `json:"spec,omitempty"` - Status *RayClusterStatusApplyConfiguration `json:"status,omitempty"` + // Specification of the desired behavior of the RayCluster. + Spec *RayClusterSpecApplyConfiguration `json:"spec,omitempty"` + Status *RayClusterStatusApplyConfiguration `json:"status,omitempty"` } // RayCluster constructs a declarative configuration of the RayCluster type for use with @@ -27,6 +31,7 @@ func RayCluster(name, namespace string) *RayClusterApplyConfiguration { b.WithAPIVersion("ray.io/v1") return b } + func (b RayClusterApplyConfiguration) IsApplyConfiguration() {} // WithKind sets the Kind field in the declarative configuration to the given value diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterspec.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterspec.go index 144ff1812d5..31c54b41405 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterspec.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterspec.go @@ -4,18 +4,38 @@ package v1 // RayClusterSpecApplyConfiguration represents a declarative configuration of the RayClusterSpec type for use // with apply. +// +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// RayClusterSpec defines the desired state of RayCluster type RayClusterSpecApplyConfiguration struct { - UpgradeStrategy *RayClusterUpgradeStrategyApplyConfiguration `json:"upgradeStrategy,omitempty"` - AuthOptions *AuthOptionsApplyConfiguration `json:"authOptions,omitempty"` - Suspend *bool `json:"suspend,omitempty"` - ManagedBy *string `json:"managedBy,omitempty"` - AutoscalerOptions *AutoscalerOptionsApplyConfiguration `json:"autoscalerOptions,omitempty"` - HeadServiceAnnotations map[string]string `json:"headServiceAnnotations,omitempty"` - EnableInTreeAutoscaling *bool `json:"enableInTreeAutoscaling,omitempty"` - GcsFaultToleranceOptions *GcsFaultToleranceOptionsApplyConfiguration `json:"gcsFaultToleranceOptions,omitempty"` - HeadGroupSpec *HeadGroupSpecApplyConfiguration `json:"headGroupSpec,omitempty"` - RayVersion *string `json:"rayVersion,omitempty"` - WorkerGroupSpecs []WorkerGroupSpecApplyConfiguration `json:"workerGroupSpecs,omitempty"` + // UpgradeStrategy defines the scaling policy used when upgrading the RayCluster + UpgradeStrategy *RayClusterUpgradeStrategyApplyConfiguration `json:"upgradeStrategy,omitempty"` + // AuthOptions specifies the authentication options for the RayCluster. + AuthOptions *AuthOptionsApplyConfiguration `json:"authOptions,omitempty"` + // Suspend indicates whether a RayCluster should be suspended. + // A suspended RayCluster will have head pods and worker pods deleted. + Suspend *bool `json:"suspend,omitempty"` + // ManagedBy is an optional configuration for the controller or entity that manages a RayCluster. + // The value must be either 'ray.io/kuberay-operator' or 'kueue.x-k8s.io/multikueue'. + // The kuberay-operator reconciles a RayCluster which doesn't have this field at all or + // the field value is the reserved string 'ray.io/kuberay-operator', + // but delegates reconciling the RayCluster with 'kueue.x-k8s.io/multikueue' to the Kueue. + // The field is immutable. + ManagedBy *string `json:"managedBy,omitempty"` + // AutoscalerOptions specifies optional configuration for the Ray autoscaler. + AutoscalerOptions *AutoscalerOptionsApplyConfiguration `json:"autoscalerOptions,omitempty"` + HeadServiceAnnotations map[string]string `json:"headServiceAnnotations,omitempty"` + // EnableInTreeAutoscaling indicates whether operator should create in tree autoscaling configs + EnableInTreeAutoscaling *bool `json:"enableInTreeAutoscaling,omitempty"` + // GcsFaultToleranceOptions for enabling GCS FT + GcsFaultToleranceOptions *GcsFaultToleranceOptionsApplyConfiguration `json:"gcsFaultToleranceOptions,omitempty"` + // HeadGroupSpec is the spec for the head pod + HeadGroupSpec *HeadGroupSpecApplyConfiguration `json:"headGroupSpec,omitempty"` + // RayVersion is used to determine the command for the Kubernetes Job managed by RayJob + RayVersion *string `json:"rayVersion,omitempty"` + // WorkerGroupSpecs are the specs for the worker pods + WorkerGroupSpecs []WorkerGroupSpecApplyConfiguration `json:"workerGroupSpecs,omitempty"` } // RayClusterSpecApplyConfiguration constructs a declarative configuration of the RayClusterSpec type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterstatus.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterstatus.go index 7e412acf58c..10d9acd9df6 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterstatus.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterstatus.go @@ -11,24 +11,56 @@ import ( // RayClusterStatusApplyConfiguration represents a declarative configuration of the RayClusterStatus type for use // with apply. +// +// RayClusterStatus defines the observed state of RayCluster type RayClusterStatusApplyConfiguration struct { - State *rayv1.ClusterState `json:"state,omitempty"` - DesiredCPU *resource.Quantity `json:"desiredCPU,omitempty"` - DesiredMemory *resource.Quantity `json:"desiredMemory,omitempty"` - DesiredGPU *resource.Quantity `json:"desiredGPU,omitempty"` - DesiredTPU *resource.Quantity `json:"desiredTPU,omitempty"` - LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` - StateTransitionTimes map[rayv1.ClusterState]*metav1.Time `json:"stateTransitionTimes,omitempty"` - Endpoints map[string]string `json:"endpoints,omitempty"` - Head *HeadInfoApplyConfiguration `json:"head,omitempty"` - Reason *string `json:"reason,omitempty"` - Conditions []applyconfigurationsmetav1.ConditionApplyConfiguration `json:"conditions,omitempty"` - ReadyWorkerReplicas *int32 `json:"readyWorkerReplicas,omitempty"` - AvailableWorkerReplicas *int32 `json:"availableWorkerReplicas,omitempty"` - DesiredWorkerReplicas *int32 `json:"desiredWorkerReplicas,omitempty"` - MinWorkerReplicas *int32 `json:"minWorkerReplicas,omitempty"` - MaxWorkerReplicas *int32 `json:"maxWorkerReplicas,omitempty"` - ObservedGeneration *int64 `json:"observedGeneration,omitempty"` + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + // Status reflects the status of the cluster + // + // Deprecated: the State field is replaced by the Conditions field. + State *rayv1.ClusterState `json:"state,omitempty"` + // DesiredCPU indicates total desired CPUs for the cluster + DesiredCPU *resource.Quantity `json:"desiredCPU,omitempty"` + // DesiredMemory indicates total desired memory for the cluster + DesiredMemory *resource.Quantity `json:"desiredMemory,omitempty"` + // DesiredGPU indicates total desired GPUs for the cluster + DesiredGPU *resource.Quantity `json:"desiredGPU,omitempty"` + // DesiredTPU indicates total desired TPUs for the cluster + DesiredTPU *resource.Quantity `json:"desiredTPU,omitempty"` + // LastUpdateTime indicates last update timestamp for this cluster status. + LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + // StateTransitionTimes indicates the time of the last state transition for each state. + StateTransitionTimes map[rayv1.ClusterState]*metav1.Time `json:"stateTransitionTimes,omitempty"` + // Service Endpoints + Endpoints map[string]string `json:"endpoints,omitempty"` + // Head info + Head *HeadInfoApplyConfiguration `json:"head,omitempty"` + // Reason provides more information about current State + Reason *string `json:"reason,omitempty"` + // Represents the latest available observations of a RayCluster's current state. + Conditions []applyconfigurationsmetav1.ConditionApplyConfiguration `json:"conditions,omitempty"` + // ReadyWorkerReplicas indicates the number of worker pods currently in the Ready state in the cluster. + // It actually reflects the number of Ready pods, although it is named "replicas" to maintain backward compatibility. + ReadyWorkerReplicas *int32 `json:"readyWorkerReplicas,omitempty"` + // AvailableWorkerReplicas indicates how many worker pods are currently available (i.e., running). + // It is named "replicas" to maintain backward compatibility. + AvailableWorkerReplicas *int32 `json:"availableWorkerReplicas,omitempty"` + // DesiredWorkerReplicas indicates the desired total number of worker Pods at the cluster level, + // calculated as the sum of `replicas * numOfHosts` for each worker group. + // It is named "replicas" to maintain backward compatibility. + DesiredWorkerReplicas *int32 `json:"desiredWorkerReplicas,omitempty"` + // MinWorkerReplicas indicates the minimum number of worker pods across all worker groups, + // calculated as the sum of `minReplicas * numOfHosts` for each worker group. + // It is named "replicas" to maintain backward compatibility. + MinWorkerReplicas *int32 `json:"minWorkerReplicas,omitempty"` + // MaxWorkerReplicas indicates the maximum number of worker pods across all worker groups, + // calculated as the sum of `maxReplicas * numOfHosts` for each worker group. + // It is named "replicas" to maintain backward compatibility. + MaxWorkerReplicas *int32 `json:"maxWorkerReplicas,omitempty"` + // observedGeneration is the most recent generation observed for this RayCluster. It corresponds to the + // RayCluster's generation, which is updated on mutation by the API Server. + ObservedGeneration *int64 `json:"observedGeneration,omitempty"` } // RayClusterStatusApplyConfiguration constructs a declarative configuration of the RayClusterStatus type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterupgradestrategy.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterupgradestrategy.go index f1a1aa39064..7dbf3799e7d 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterupgradestrategy.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayclusterupgradestrategy.go @@ -9,6 +9,7 @@ import ( // RayClusterUpgradeStrategyApplyConfiguration represents a declarative configuration of the RayClusterUpgradeStrategy type for use // with apply. type RayClusterUpgradeStrategyApplyConfiguration struct { + // Type represents the strategy used when upgrading the RayCluster Pods. Currently supports `Recreate` and `None`. Type *rayv1.RayClusterUpgradeType `json:"type,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjob.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjob.go index deeb9015c76..882057ea289 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjob.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjob.go @@ -10,6 +10,8 @@ import ( // RayJobApplyConfiguration represents a declarative configuration of the RayJob type for use // with apply. +// +// RayJob is the Schema for the rayjobs API type RayJobApplyConfiguration struct { metav1.TypeMetaApplyConfiguration `json:",inline"` *metav1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` @@ -27,6 +29,7 @@ func RayJob(name, namespace string) *RayJobApplyConfiguration { b.WithAPIVersion("ray.io/v1") return b } + func (b RayJobApplyConfiguration) IsApplyConfiguration() {} // WithKind sets the Kind field in the declarative configuration to the given value diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go index 14083877a44..60d69094112 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go @@ -9,26 +9,70 @@ import ( // RayJobSpecApplyConfiguration represents a declarative configuration of the RayJobSpec type for use // with apply. +// +// RayJobSpec defines the desired state of RayJob type RayJobSpecApplyConfiguration struct { - ActiveDeadlineSeconds *int32 `json:"activeDeadlineSeconds,omitempty"` - BackoffLimit *int32 `json:"backoffLimit,omitempty"` - RayClusterSpec *RayClusterSpecApplyConfiguration `json:"rayClusterSpec,omitempty"` - SubmitterPodTemplate *corev1.PodTemplateSpecApplyConfiguration `json:"submitterPodTemplate,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` - ClusterSelector map[string]string `json:"clusterSelector,omitempty"` - SubmitterConfig *SubmitterConfigApplyConfiguration `json:"submitterConfig,omitempty"` - ManagedBy *string `json:"managedBy,omitempty"` - DeletionStrategy *DeletionStrategyApplyConfiguration `json:"deletionStrategy,omitempty"` - Entrypoint *string `json:"entrypoint,omitempty"` - RuntimeEnvYAML *string `json:"runtimeEnvYAML,omitempty"` - JobId *string `json:"jobId,omitempty"` - SubmissionMode *rayv1.JobSubmissionMode `json:"submissionMode,omitempty"` - EntrypointResources *string `json:"entrypointResources,omitempty"` - EntrypointNumCpus *float32 `json:"entrypointNumCpus,omitempty"` - EntrypointNumGpus *float32 `json:"entrypointNumGpus,omitempty"` - TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"` - ShutdownAfterJobFinishes *bool `json:"shutdownAfterJobFinishes,omitempty"` - Suspend *bool `json:"suspend,omitempty"` + // ActiveDeadlineSeconds is the duration in seconds that the RayJob may be active before + // KubeRay actively tries to terminate the RayJob; value must be positive integer. + ActiveDeadlineSeconds *int32 `json:"activeDeadlineSeconds,omitempty"` + // Specifies the number of retries before marking this job failed. + // Each retry creates a new RayCluster. + BackoffLimit *int32 `json:"backoffLimit,omitempty"` + // RayClusterSpec is the cluster template to run the job + RayClusterSpec *RayClusterSpecApplyConfiguration `json:"rayClusterSpec,omitempty"` + // SubmitterPodTemplate is the template for the pod that will run `ray job submit`. + SubmitterPodTemplate *corev1.PodTemplateSpecApplyConfiguration `json:"submitterPodTemplate,omitempty"` + // Metadata is data to store along with this job. + Metadata map[string]string `json:"metadata,omitempty"` + // clusterSelector is used to select running rayclusters by labels + ClusterSelector map[string]string `json:"clusterSelector,omitempty"` + // Configurations of submitter k8s job. + SubmitterConfig *SubmitterConfigApplyConfiguration `json:"submitterConfig,omitempty"` + // ManagedBy is an optional configuration for the controller or entity that manages a RayJob. + // The value must be either 'ray.io/kuberay-operator' or 'kueue.x-k8s.io/multikueue'. + // The kuberay-operator reconciles a RayJob which doesn't have this field at all or + // the field value is the reserved string 'ray.io/kuberay-operator', + // but delegates reconciling the RayJob with 'kueue.x-k8s.io/multikueue' to the Kueue. + // The field is immutable. + ManagedBy *string `json:"managedBy,omitempty"` + // DeletionStrategy automates post-completion cleanup. + // Choose one style or omit: + // - Legacy: both onSuccess & onFailure (deprecated; may combine with shutdownAfterJobFinishes and TTLSecondsAfterFinished). + // - Rules: deletionRules (non-empty) — incompatible with shutdownAfterJobFinishes, legacy fields, and global TTLSecondsAfterFinished (use per-rule condition.ttlSeconds). + // Global TTLSecondsAfterFinished > 0 requires shutdownAfterJobFinishes=true. + // Feature gate RayJobDeletionPolicy must be enabled when this field is set. + DeletionStrategy *DeletionStrategyApplyConfiguration `json:"deletionStrategy,omitempty"` + // Entrypoint represents the command to start execution. + Entrypoint *string `json:"entrypoint,omitempty"` + // RuntimeEnvYAML represents the runtime environment configuration + // provided as a multi-line YAML string. + RuntimeEnvYAML *string `json:"runtimeEnvYAML,omitempty"` + // If jobId is not set, a new jobId will be auto-generated. + JobId *string `json:"jobId,omitempty"` + // SubmissionMode specifies how RayJob submits the Ray job to the RayCluster. + // In "K8sJobMode", the KubeRay operator creates a submitter Kubernetes Job to submit the Ray job. + // In "HTTPMode", the KubeRay operator sends a request to the RayCluster to create a Ray job. + // In "InteractiveMode", the KubeRay operator waits for a user to submit a job to the Ray cluster. + // In "SidecarMode", the KubeRay operator injects a container into the Ray head Pod that acts as the job submitter to submit the Ray job. + SubmissionMode *rayv1.JobSubmissionMode `json:"submissionMode,omitempty"` + // EntrypointResources specifies the custom resources and quantities to reserve for the + // entrypoint command. + EntrypointResources *string `json:"entrypointResources,omitempty"` + // EntrypointNumCpus specifies the number of cpus to reserve for the entrypoint command. + EntrypointNumCpus *float32 `json:"entrypointNumCpus,omitempty"` + // EntrypointNumGpus specifies the number of gpus to reserve for the entrypoint command. + EntrypointNumGpus *float32 `json:"entrypointNumGpus,omitempty"` + // TTLSecondsAfterFinished is the TTL to clean up RayCluster. + // It's only working when ShutdownAfterJobFinishes set to true. + TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"` + // ShutdownAfterJobFinishes will determine whether to delete the ray cluster once rayJob succeed or failed. + ShutdownAfterJobFinishes *bool `json:"shutdownAfterJobFinishes,omitempty"` + // suspend specifies whether the RayJob controller should create a RayCluster instance + // If a job is applied with the suspend field set to true, + // the RayCluster will not be created and will wait for the transition to false. + // If the RayCluster is already created, it will be deleted. + // In case of transition to false a new RayCluster will be created. + Suspend *bool `json:"suspend,omitempty"` } // RayJobSpecApplyConfiguration constructs a declarative configuration of the RayJobSpec type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatus.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatus.go index de1e5fdba42..61baad836f6 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatus.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatus.go @@ -9,7 +9,12 @@ import ( // RayJobStatusApplyConfiguration represents a declarative configuration of the RayJobStatus type for use // with apply. +// +// RayJobStatus defines the observed state of RayJob type RayJobStatusApplyConfiguration struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + // RayJobStatusInfo contains information about the Ray job retrieved from the Ray dashboard. RayJobStatusInfo *RayJobStatusInfoApplyConfiguration `json:"rayJobInfo,omitempty"` JobId *string `json:"jobId,omitempty"` RayClusterName *string `json:"rayClusterName,omitempty"` @@ -18,12 +23,21 @@ type RayJobStatusApplyConfiguration struct { JobDeploymentStatus *rayv1.JobDeploymentStatus `json:"jobDeploymentStatus,omitempty"` Reason *rayv1.JobFailedReason `json:"reason,omitempty"` Message *string `json:"message,omitempty"` - StartTime *metav1.Time `json:"startTime,omitempty"` - EndTime *metav1.Time `json:"endTime,omitempty"` - Succeeded *int32 `json:"succeeded,omitempty"` - Failed *int32 `json:"failed,omitempty"` - RayClusterStatus *RayClusterStatusApplyConfiguration `json:"rayClusterStatus,omitempty"` - ObservedGeneration *int64 `json:"observedGeneration,omitempty"` + // StartTime is the time when JobDeploymentStatus transitioned from 'New' to 'Initializing'. + StartTime *metav1.Time `json:"startTime,omitempty"` + // EndTime is the time when JobDeploymentStatus transitioned to 'Complete' status. + // This occurs when the Ray job reaches a terminal state (SUCCEEDED, FAILED, STOPPED) + // or the submitter Job has failed. + EndTime *metav1.Time `json:"endTime,omitempty"` + // Succeeded is the number of times this job succeeded. + Succeeded *int32 `json:"succeeded,omitempty"` + // Failed is the number of times this job failed. + Failed *int32 `json:"failed,omitempty"` + // RayClusterStatus is the status of the RayCluster running the job. + RayClusterStatus *RayClusterStatusApplyConfiguration `json:"rayClusterStatus,omitempty"` + // observedGeneration is the most recent generation observed for this RayJob. It corresponds to the + // RayJob's generation, which is updated on mutation by the API Server. + ObservedGeneration *int64 `json:"observedGeneration,omitempty"` } // RayJobStatusApplyConfiguration constructs a declarative configuration of the RayJobStatus type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatusinfo.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatusinfo.go index 64d76e8fcf4..ba5529ce24b 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatusinfo.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobstatusinfo.go @@ -8,6 +8,12 @@ import ( // RayJobStatusInfoApplyConfiguration represents a declarative configuration of the RayJobStatusInfo type for use // with apply. +// +// `RayJobStatusInfo` is a subset of `RayJobInfo` from `dashboard_httpclient.py`. +// This subset is used to store information in the CR status. +// +// TODO(kevin85421): We can consider exposing the whole `RayJobInfo` in the CR status +// after careful consideration. In that case, we can remove `RayJobStatusInfo`. type RayJobStatusInfoApplyConfiguration struct { StartTime *metav1.Time `json:"startTime,omitempty"` EndTime *metav1.Time `json:"endTime,omitempty"` diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservice.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservice.go index 1ca30cc98dc..fa7b45729bd 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservice.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservice.go @@ -10,6 +10,8 @@ import ( // RayServiceApplyConfiguration represents a declarative configuration of the RayService type for use // with apply. +// +// RayService is the Schema for the rayservices API type RayServiceApplyConfiguration struct { metav1.TypeMetaApplyConfiguration `json:",inline"` *metav1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` @@ -27,6 +29,7 @@ func RayService(name, namespace string) *RayServiceApplyConfiguration { b.WithAPIVersion("ray.io/v1") return b } + func (b RayServiceApplyConfiguration) IsApplyConfiguration() {} // WithKind sets the Kind field in the declarative configuration to the given value diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicespec.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicespec.go index faef5106dfe..79548b0a094 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicespec.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicespec.go @@ -8,15 +8,27 @@ import ( // RayServiceSpecApplyConfiguration represents a declarative configuration of the RayServiceSpec type for use // with apply. +// +// RayServiceSpec defines the desired state of RayService type RayServiceSpecApplyConfiguration struct { - RayClusterDeletionDelaySeconds *int32 `json:"rayClusterDeletionDelaySeconds,omitempty"` - ServiceUnhealthySecondThreshold *int32 `json:"serviceUnhealthySecondThreshold,omitempty"` - DeploymentUnhealthySecondThreshold *int32 `json:"deploymentUnhealthySecondThreshold,omitempty"` - ServeService *corev1.Service `json:"serveService,omitempty"` - UpgradeStrategy *RayServiceUpgradeStrategyApplyConfiguration `json:"upgradeStrategy,omitempty"` - ServeConfigV2 *string `json:"serveConfigV2,omitempty"` - RayClusterSpec *RayClusterSpecApplyConfiguration `json:"rayClusterConfig,omitempty"` - ExcludeHeadPodFromServeSvc *bool `json:"excludeHeadPodFromServeSvc,omitempty"` + // RayClusterDeletionDelaySeconds specifies the delay, in seconds, before deleting old RayClusters. + // The default value is 60 seconds. + RayClusterDeletionDelaySeconds *int32 `json:"rayClusterDeletionDelaySeconds,omitempty"` + // Deprecated: This field is not used anymore. ref: https://github.com/ray-project/kuberay/issues/1685 + ServiceUnhealthySecondThreshold *int32 `json:"serviceUnhealthySecondThreshold,omitempty"` + // Deprecated: This field is not used anymore. ref: https://github.com/ray-project/kuberay/issues/1685 + DeploymentUnhealthySecondThreshold *int32 `json:"deploymentUnhealthySecondThreshold,omitempty"` + // ServeService is the Kubernetes service for head node and worker nodes who have healthy http proxy to serve traffics. + ServeService *corev1.Service `json:"serveService,omitempty"` + // UpgradeStrategy defines the scaling policy used when upgrading the RayService. + UpgradeStrategy *RayServiceUpgradeStrategyApplyConfiguration `json:"upgradeStrategy,omitempty"` + // Important: Run "make" to regenerate code after modifying this file + // Defines the applications and deployments to deploy, should be a YAML multi-line scalar string. + ServeConfigV2 *string `json:"serveConfigV2,omitempty"` + RayClusterSpec *RayClusterSpecApplyConfiguration `json:"rayClusterConfig,omitempty"` + // If the field is set to true, the value of the label `ray.io/serve` on the head Pod should always be false. + // Therefore, the head Pod's endpoint will not be added to the Kubernetes Serve service. + ExcludeHeadPodFromServeSvc *bool `json:"excludeHeadPodFromServeSvc,omitempty"` } // RayServiceSpecApplyConfiguration constructs a declarative configuration of the RayServiceSpec type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatus.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatus.go index 2d7f2984cef..a95ba6f97fe 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatus.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatus.go @@ -9,12 +9,21 @@ import ( // RayServiceStatusApplyConfiguration represents a declarative configuration of the RayServiceStatus type for use // with apply. type RayServiceStatusApplyConfiguration struct { - Applications map[string]AppStatusApplyConfiguration `json:"applicationStatuses,omitempty"` - TargetCapacity *int32 `json:"targetCapacity,omitempty"` - TrafficRoutedPercent *int32 `json:"trafficRoutedPercent,omitempty"` - LastTrafficMigratedTime *metav1.Time `json:"lastTrafficMigratedTime,omitempty"` - RayClusterName *string `json:"rayClusterName,omitempty"` - RayClusterStatus *RayClusterStatusApplyConfiguration `json:"rayClusterStatus,omitempty"` + // Important: Run "make" to regenerate code after modifying this file + Applications map[string]AppStatusApplyConfiguration `json:"applicationStatuses,omitempty"` + // TargetCapacity is the `target_capacity` percentage for all Serve replicas + // across the cluster for this RayService. The `num_replicas`, `min_replicas`, `max_replicas`, + // and `initial_replicas` for each deployment will be scaled by this percentage." + TargetCapacity *int32 `json:"targetCapacity,omitempty"` + // TrafficRoutedPercent is the percentage of traffic that is routed to the Serve service + // for this RayService. TrafficRoutedPercent is updated to reflect the weight on the HTTPRoute + // created for this RayService during incremental upgrades to a new cluster. + TrafficRoutedPercent *int32 `json:"trafficRoutedPercent,omitempty"` + // LastTrafficMigratedTime is the last time that TrafficRoutedPercent was updated to a new value + // for this RayService. + LastTrafficMigratedTime *metav1.Time `json:"lastTrafficMigratedTime,omitempty"` + RayClusterName *string `json:"rayClusterName,omitempty"` + RayClusterStatus *RayClusterStatusApplyConfiguration `json:"rayClusterStatus,omitempty"` } // RayServiceStatusApplyConfiguration constructs a declarative configuration of the RayServiceStatus type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatuses.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatuses.go index 7d0da98387f..b6895e89648 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatuses.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayservicestatuses.go @@ -10,14 +10,26 @@ import ( // RayServiceStatusesApplyConfiguration represents a declarative configuration of the RayServiceStatuses type for use // with apply. +// +// RayServiceStatuses defines the observed state of RayService type RayServiceStatusesApplyConfiguration struct { - Conditions []metav1.ConditionApplyConfiguration `json:"conditions,omitempty"` - LastUpdateTime *apismetav1.Time `json:"lastUpdateTime,omitempty"` - ServiceStatus *rayv1.ServiceStatus `json:"serviceStatus,omitempty"` - ActiveServiceStatus *RayServiceStatusApplyConfiguration `json:"activeServiceStatus,omitempty"` - PendingServiceStatus *RayServiceStatusApplyConfiguration `json:"pendingServiceStatus,omitempty"` - NumServeEndpoints *int32 `json:"numServeEndpoints,omitempty"` - ObservedGeneration *int64 `json:"observedGeneration,omitempty"` + // Represents the latest available observations of a RayService's current state. + Conditions []metav1.ConditionApplyConfiguration `json:"conditions,omitempty"` + // LastUpdateTime represents the timestamp when the RayService status was last updated. + LastUpdateTime *apismetav1.Time `json:"lastUpdateTime,omitempty"` + // Deprecated: `ServiceStatus` is deprecated - use `Conditions` instead. `Running` means the RayService is ready to + // serve requests. An empty `ServiceStatus` means the RayService is not ready to serve requests. The definition of + // `ServiceStatus` is equivalent to the `RayServiceReady` condition. + ServiceStatus *rayv1.ServiceStatus `json:"serviceStatus,omitempty"` + ActiveServiceStatus *RayServiceStatusApplyConfiguration `json:"activeServiceStatus,omitempty"` + // Pending Service Status indicates a RayCluster will be created or is being created. + PendingServiceStatus *RayServiceStatusApplyConfiguration `json:"pendingServiceStatus,omitempty"` + // NumServeEndpoints indicates the number of Ray Pods that are actively serving or have been selected by the serve service. + // Ray Pods without a proxy actor or those that are unhealthy will not be counted. + NumServeEndpoints *int32 `json:"numServeEndpoints,omitempty"` + // observedGeneration is the most recent generation observed for this RayService. It corresponds to the + // RayService's generation, which is updated on mutation by the API Server. + ObservedGeneration *int64 `json:"observedGeneration,omitempty"` } // RayServiceStatusesApplyConfiguration constructs a declarative configuration of the RayServiceStatuses type for use with diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayserviceupgradestrategy.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayserviceupgradestrategy.go index c8cfc02aed6..e69ac81bfcf 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayserviceupgradestrategy.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayserviceupgradestrategy.go @@ -9,7 +9,10 @@ import ( // RayServiceUpgradeStrategyApplyConfiguration represents a declarative configuration of the RayServiceUpgradeStrategy type for use // with apply. type RayServiceUpgradeStrategyApplyConfiguration struct { - Type *rayv1.RayServiceUpgradeType `json:"type,omitempty"` + // Type represents the strategy used when upgrading the RayService. Currently supports `NewCluster`, `NewClusterWithIncrementalUpgrade` and `None`. + Type *rayv1.RayServiceUpgradeType `json:"type,omitempty"` + // ClusterUpgradeOptions defines the behavior of a NewClusterWithIncrementalUpgrade type. + // RayServiceIncrementalUpgrade feature gate must be enabled to set ClusterUpgradeOptions. ClusterUpgradeOptions *ClusterUpgradeOptionsApplyConfiguration `json:"clusterUpgradeOptions,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rediscredential.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rediscredential.go index 6afcbe2022d..1cd15caea76 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rediscredential.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rediscredential.go @@ -8,6 +8,8 @@ import ( // RedisCredentialApplyConfiguration represents a declarative configuration of the RedisCredential type for use // with apply. +// +// RedisCredential is the redis username/password or a reference to the source containing the username/password type RedisCredentialApplyConfiguration struct { ValueFrom *corev1.EnvVarSource `json:"valueFrom,omitempty"` Value *string `json:"value,omitempty"` diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go index b159c29f926..7ce8ee06ad6 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go @@ -4,7 +4,10 @@ package v1 // ScaleStrategyApplyConfiguration represents a declarative configuration of the ScaleStrategy type for use // with apply. +// +// ScaleStrategy to remove workers type ScaleStrategyApplyConfiguration struct { + // WorkersToDelete workers to be deleted WorkersToDelete []string `json:"workersToDelete,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/servedeploymentstatus.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/servedeploymentstatus.go index 94872a933c6..29588d0e9d5 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/servedeploymentstatus.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/servedeploymentstatus.go @@ -4,6 +4,8 @@ package v1 // ServeDeploymentStatusApplyConfiguration represents a declarative configuration of the ServeDeploymentStatus type for use // with apply. +// +// ServeDeploymentStatus defines the current state of a Serve deployment type ServeDeploymentStatusApplyConfiguration struct { Status *string `json:"status,omitempty"` Message *string `json:"message,omitempty"` diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/submitterconfig.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/submitterconfig.go index 2992e56d6ff..020d1b0944e 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/submitterconfig.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/submitterconfig.go @@ -5,6 +5,7 @@ package v1 // SubmitterConfigApplyConfiguration represents a declarative configuration of the SubmitterConfig type for use // with apply. type SubmitterConfigApplyConfiguration struct { + // BackoffLimit of the submitter k8s job. BackoffLimit *int32 `json:"backoffLimit,omitempty"` } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/workergroupspec.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/workergroupspec.go index 7aa692895f1..25605e8ecdb 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/workergroupspec.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/workergroupspec.go @@ -8,19 +8,40 @@ import ( // WorkerGroupSpecApplyConfiguration represents a declarative configuration of the WorkerGroupSpec type for use // with apply. +// +// WorkerGroupSpec are the specs for the worker pods type WorkerGroupSpecApplyConfiguration struct { - Suspend *bool `json:"suspend,omitempty"` - GroupName *string `json:"groupName,omitempty"` - Replicas *int32 `json:"replicas,omitempty"` - MinReplicas *int32 `json:"minReplicas,omitempty"` - MaxReplicas *int32 `json:"maxReplicas,omitempty"` - IdleTimeoutSeconds *int32 `json:"idleTimeoutSeconds,omitempty"` - Resources map[string]string `json:"resources,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - RayStartParams map[string]string `json:"rayStartParams,omitempty"` - Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` - ScaleStrategy *ScaleStrategyApplyConfiguration `json:"scaleStrategy,omitempty"` - NumOfHosts *int32 `json:"numOfHosts,omitempty"` + // Suspend indicates whether a worker group should be suspended. + // A suspended worker group will have all pods deleted. + // This is not a user-facing API and is only used by RayJob DeletionStrategy. + Suspend *bool `json:"suspend,omitempty"` + // we can have multiple worker groups, we distinguish them by name + GroupName *string `json:"groupName,omitempty"` + // Replicas is the number of desired Pods for this worker group. See https://github.com/ray-project/kuberay/pull/1443 for more details about the reason for making this field optional. + Replicas *int32 `json:"replicas,omitempty"` + // MinReplicas denotes the minimum number of desired Pods for this worker group. + MinReplicas *int32 `json:"minReplicas,omitempty"` + // MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32. + MaxReplicas *int32 `json:"maxReplicas,omitempty"` + // IdleTimeoutSeconds denotes the number of seconds to wait before the v2 autoscaler terminates an idle worker pod of this type. + // This value is only used with the Ray Autoscaler enabled and defaults to the value set by the AutoscalingConfig if not specified for this worker group. + IdleTimeoutSeconds *int32 `json:"idleTimeoutSeconds,omitempty"` + // Resources specifies the resource quantities for this worker group. + // These values override the resources passed to `rayStartParams` for the group, but + // have no effect on the resources set at the K8s Pod container level. + Resources map[string]string `json:"resources,omitempty"` + // Labels specifies the Ray node labels for this worker group. + // These labels will also be added to the Pods of this worker group and override the `--labels` + // argument passed to `rayStartParams`. + Labels map[string]string `json:"labels,omitempty"` + // RayStartParams are the params of the start command: address, object-store-memory, ... + RayStartParams map[string]string `json:"rayStartParams,omitempty"` + // Template is a pod template for the worker + Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` + // ScaleStrategy defines which pods to remove + ScaleStrategy *ScaleStrategyApplyConfiguration `json:"scaleStrategy,omitempty"` + // NumOfHosts denotes the number of hosts to create per replica. The default value is 1. + NumOfHosts *int32 `json:"numOfHosts,omitempty"` } // WorkerGroupSpecApplyConfiguration constructs a declarative configuration of the WorkerGroupSpec type for use with diff --git a/ray-operator/pkg/client/clientset/versioned/fake/clientset_generated.go b/ray-operator/pkg/client/clientset/versioned/fake/clientset_generated.go index 9c85773ad3f..3a09ed30290 100644 --- a/ray-operator/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/ray-operator/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -20,7 +20,7 @@ import ( // without applying any field management, validations and/or defaults. It shouldn't be considered a replacement // for a real clientset and is mostly useful in simple unit tests. // -// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// Deprecated: NewClientset replaces this with support for field management, which significantly improves // server side apply testing. NewClientset is only available when apply configurations are generated (e.g. // via --with-applyconfig). func NewSimpleClientset(objects ...runtime.Object) *Clientset { @@ -36,8 +36,8 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset { cs.AddReactor("*", "*", testing.ObjectReaction(o)) cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { var opts metav1.ListOptions - if watchActcion, ok := action.(testing.WatchActionImpl); ok { - opts = watchActcion.ListOptions + if watchAction, ok := action.(testing.WatchActionImpl); ok { + opts = watchAction.ListOptions } gvr := action.GetResource() ns := action.GetNamespace() @@ -68,6 +68,17 @@ func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } +// IsWatchListSemanticsSupported informs the reflector that this client +// doesn't support WatchList semantics. +// +// This is a synthetic method whose sole purpose is to satisfy the optional +// interface check performed by the reflector. +// Returning true signals that WatchList can NOT be used. +// No additional logic is implemented here. +func (c *Clientset) IsWatchListSemanticsUnSupported() bool { + return true +} + // NewClientset returns a clientset that will respond with the provided objects. // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, // without applying any validations and/or defaults. It shouldn't be considered a replacement diff --git a/ray-operator/pkg/client/informers/externalversions/factory.go b/ray-operator/pkg/client/informers/externalversions/factory.go index d909b4db15b..3e483db43c3 100644 --- a/ray-operator/pkg/client/informers/externalversions/factory.go +++ b/ray-operator/pkg/client/informers/externalversions/factory.go @@ -81,6 +81,7 @@ func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Dur // NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. // Listers obtained via this SharedInformerFactory will be subject to the same filters // as specified here. +// // Deprecated: Please use NewSharedInformerFactoryWithOptions instead func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) @@ -188,7 +189,7 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal // // It is typically used like this: // -// ctx, cancel := context.Background() +// ctx, cancel := context.WithCancel(context.Background()) // defer cancel() // factory := NewSharedInformerFactory(client, resyncPeriod) // defer factory.WaitForStop() // Returns immediately if nothing was started. diff --git a/ray-operator/pkg/client/informers/externalversions/ray/v1/raycluster.go b/ray-operator/pkg/client/informers/externalversions/ray/v1/raycluster.go index 33d0ee56143..6937344b296 100644 --- a/ray-operator/pkg/client/informers/externalversions/ray/v1/raycluster.go +++ b/ray-operator/pkg/client/informers/externalversions/ray/v1/raycluster.go @@ -41,7 +41,7 @@ func NewRayClusterInformer(client versioned.Interface, namespace string, resyncP // one. This reduces memory footprint and number of connections to the server. func NewFilteredRayClusterInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( - &cache.ListWatch{ + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) @@ -66,7 +66,7 @@ func NewFilteredRayClusterInformer(client versioned.Interface, namespace string, } return client.RayV1().RayClusters(namespace).Watch(ctx, options) }, - }, + }, client), &apisrayv1.RayCluster{}, resyncPeriod, indexers, diff --git a/ray-operator/pkg/client/informers/externalversions/ray/v1/rayjob.go b/ray-operator/pkg/client/informers/externalversions/ray/v1/rayjob.go index 11220f7f376..a6febf4894c 100644 --- a/ray-operator/pkg/client/informers/externalversions/ray/v1/rayjob.go +++ b/ray-operator/pkg/client/informers/externalversions/ray/v1/rayjob.go @@ -41,7 +41,7 @@ func NewRayJobInformer(client versioned.Interface, namespace string, resyncPerio // one. This reduces memory footprint and number of connections to the server. func NewFilteredRayJobInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( - &cache.ListWatch{ + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) @@ -66,7 +66,7 @@ func NewFilteredRayJobInformer(client versioned.Interface, namespace string, res } return client.RayV1().RayJobs(namespace).Watch(ctx, options) }, - }, + }, client), &apisrayv1.RayJob{}, resyncPeriod, indexers, diff --git a/ray-operator/pkg/client/informers/externalversions/ray/v1/rayservice.go b/ray-operator/pkg/client/informers/externalversions/ray/v1/rayservice.go index f2aa54d88ea..9a3a9d48ef2 100644 --- a/ray-operator/pkg/client/informers/externalversions/ray/v1/rayservice.go +++ b/ray-operator/pkg/client/informers/externalversions/ray/v1/rayservice.go @@ -41,7 +41,7 @@ func NewRayServiceInformer(client versioned.Interface, namespace string, resyncP // one. This reduces memory footprint and number of connections to the server. func NewFilteredRayServiceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( - &cache.ListWatch{ + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) @@ -66,7 +66,7 @@ func NewFilteredRayServiceInformer(client versioned.Interface, namespace string, } return client.RayV1().RayServices(namespace).Watch(ctx, options) }, - }, + }, client), &apisrayv1.RayService{}, resyncPeriod, indexers,