From 9f5842e71b36d0580e6b4989fadf2c700a698806 Mon Sep 17 00:00:00 2001 From: rockerchen Date: Wed, 18 Sep 2024 14:56:29 +0800 Subject: [PATCH 1/2] add tke-extend-network-controller v1.1.0 --- .../tke-extend-network-controller/.helmignore | 23 +++ .../tke-extend-network-controller/Chart.yaml | 33 ++++ .../tke-extend-network-controller/README.md | 44 +++++ .../templates/_helpers.tpl | 62 ++++++ .../templates/cert.yaml | 19 ++ .../templates/deployment.yaml | 83 ++++++++ .../templates/issuer.yaml | 10 + ...oud.tencent.com_dedicatedclblisteners.yaml | 146 ++++++++++++++ ...loud.tencent.com_dedicatedclbservices.yaml | 152 ++++++++++++++ .../templates/role.yaml | 186 ++++++++++++++++++ .../templates/rolebinding.yaml | 45 +++++ .../templates/secret-env.yaml | 16 ++ .../templates/service.yaml | 30 +++ .../templates/serviceaccount.yaml | 8 + .../templates/webhook.yaml | 62 ++++++ .../tke-extend-network-controller/values.yaml | 73 +++++++ 16 files changed, 992 insertions(+) create mode 100644 incubator/tke-extend-network-controller/.helmignore create mode 100644 incubator/tke-extend-network-controller/Chart.yaml create mode 100644 incubator/tke-extend-network-controller/README.md create mode 100644 incubator/tke-extend-network-controller/templates/_helpers.tpl create mode 100644 incubator/tke-extend-network-controller/templates/cert.yaml create mode 100644 incubator/tke-extend-network-controller/templates/deployment.yaml create mode 100644 incubator/tke-extend-network-controller/templates/issuer.yaml create mode 100644 incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclblisteners.yaml create mode 100644 incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclbservices.yaml create mode 100644 incubator/tke-extend-network-controller/templates/role.yaml create mode 100644 incubator/tke-extend-network-controller/templates/rolebinding.yaml create mode 100644 incubator/tke-extend-network-controller/templates/secret-env.yaml create mode 100644 incubator/tke-extend-network-controller/templates/service.yaml create mode 100644 incubator/tke-extend-network-controller/templates/serviceaccount.yaml create mode 100644 incubator/tke-extend-network-controller/templates/webhook.yaml create mode 100644 incubator/tke-extend-network-controller/values.yaml diff --git a/incubator/tke-extend-network-controller/.helmignore b/incubator/tke-extend-network-controller/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/incubator/tke-extend-network-controller/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/incubator/tke-extend-network-controller/Chart.yaml b/incubator/tke-extend-network-controller/Chart.yaml new file mode 100644 index 00000000..d90837c9 --- /dev/null +++ b/incubator/tke-extend-network-controller/Chart.yaml @@ -0,0 +1,33 @@ +apiVersion: v2 +name: tke-extend-network-controller +description: A Network Controller for TKE + +home: https://github.com/imroc/tke-extend-network-controller +icon: https://cloudcache.tencent-cloud.com/qcloud/ui/static/Industry_tke/b76ab4be-8275-4f41-8438-764267b3af73.png + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.1.0" + +maintainers: + - name: rockerchen + email: rockerchen@tencent.com + +kubeVerion: ">= 1.26.0" diff --git a/incubator/tke-extend-network-controller/README.md b/incubator/tke-extend-network-controller/README.md new file mode 100644 index 00000000..2f032c9d --- /dev/null +++ b/incubator/tke-extend-network-controller/README.md @@ -0,0 +1,44 @@ +# tke-extend-network-controller + +![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square) + +A Network Controller for TKE + +**Homepage:** + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| clusterID | string | `""` | Cluster ID of the current TKE Cluster. | +| fullnameOverride | string | `""` | | +| image | object | `{"pullPolicy":"IfNotPresent","repository":"imroc/tke-extend-network-controller","tag":""}` | Image of the controller | +| image.pullPolicy | string | `"IfNotPresent"` | ImagePullPolicy of the controller | +| image.repository | string | `"imroc/tke-extend-network-controller"` | Image repository of the controller | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | ImagePullSecrets of the controller | +| livenessProbe.httpGet.path | string | `"/healthz"` | | +| livenessProbe.httpGet.port | int | `8081` | | +| livenessProbe.initialDelaySeconds | int | `15` | | +| livenessProbe.periodSeconds | int | `20` | | +| log | object | `{"encoder":"json","level":"info"}` | Logging otpions of the controller | +| log.encoder | string | `"json"` | Log format of the controller, be one of 'json' or 'console' | +| log.level | string | `"info"` | Log level of the controller, be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| readinessProbe.httpGet.path | string | `"/readyz"` | | +| readinessProbe.httpGet.port | int | `8081` | | +| readinessProbe.initialDelaySeconds | int | `5` | | +| readinessProbe.periodSeconds | int | `10` | | +| region | string | `""` | Region of the current TKE Cluster, optional. | +| replicaCount | int | `1` | Replica count of the controller pod | +| resources | object | `{}` | | +| secretID | string | `""` | Secret ID of the Tencent Cloud Account. | +| secretKey | string | `""` | Secret Key of the Tencent Cloud Account. | +| tolerations | list | `[]` | | +| vpcID | string | `""` | VPC ID of the current TKE Cluster. | +| workerCount | int | `3` | Worker count of the controller | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/incubator/tke-extend-network-controller/templates/_helpers.tpl b/incubator/tke-extend-network-controller/templates/_helpers.tpl new file mode 100644 index 00000000..0ff4f4ad --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "tke-extend-network-controller.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "tke-extend-network-controller.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "tke-extend-network-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "tke-extend-network-controller.labels" -}} +helm.sh/chart: {{ include "tke-extend-network-controller.chart" . }} +{{ include "tke-extend-network-controller.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "tke-extend-network-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "tke-extend-network-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "tke-extend-network-controller.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "tke-extend-network-controller.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/incubator/tke-extend-network-controller/templates/cert.yaml b/incubator/tke-extend-network-controller/templates/cert.yaml new file mode 100644 index 00000000..d04636db --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/cert.yaml @@ -0,0 +1,19 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: {{ include "tke-extend-network-controller.fullname" . }} + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: {{ include "tke-extend-network-controller.fullname" . }} + name: {{ include "tke-extend-network-controller.fullname" . }}-serving-cert + namespace: {{ .Release.Namespace | quote }} +spec: + dnsNames: + - {{ include "tke-extend-network-controller.fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc + - {{ include "tke-extend-network-controller.fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "tke-extend-network-controller.fullname" . }}-selfsigned-issuer + secretName: {{ include "tke-extend-network-controller.fullname" . }}-webhook-server-cert diff --git a/incubator/tke-extend-network-controller/templates/deployment.yaml b/incubator/tke-extend-network-controller/templates/deployment.yaml new file mode 100644 index 00000000..2910960a --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/deployment.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }} + namespace: {{ .Release.Namespace | quote }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "tke-extend-network-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: controller + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 8 }} + spec: + hostAliases: + - hostnames: + - clb.tencentcloudapi.com + ip: 169.254.0.95 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - args: + - --metrics-bind-address=:8443 + - --leader-elect + - --health-probe-bind-address=:8081 + - --worker-count={{ .Values.workerCount }} + - --zap-log-level={{ .Values.log.level }} + - --zap-encoder={{ .Values.log.encoder }} + command: + - /tke-extend-network-controller + envFrom: + - secretRef: + name: {{ include "tke-extend-network-controller.fullname" . }}-env + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: controller + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 10 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 10 }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + runAsNonRoot: true + serviceAccountName: {{ include "tke-extend-network-controller.fullname" . }} + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: {{ include "tke-extend-network-controller.fullname" . }}-webhook-server-cert + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/incubator/tke-extend-network-controller/templates/issuer.yaml b/incubator/tke-extend-network-controller/templates/issuer.yaml new file mode 100644 index 00000000..abcb84c8 --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/issuer.yaml @@ -0,0 +1,10 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: {{ include "tke-extend-network-controller.fullname" . }} + name: {{ include "tke-extend-network-controller.fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace | quote }} +spec: + selfSigned: {} + diff --git a/incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclblisteners.yaml b/incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclblisteners.yaml new file mode 100644 index 00000000..de503fc2 --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclblisteners.yaml @@ -0,0 +1,146 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "tke-extend-network-controller.fullname" . }}-serving-cert + name: dedicatedclblisteners.networking.cloud.tencent.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: {{ include "tke-extend-network-controller.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace | quote }} + path: /convert + conversionReviewVersions: + - v1 + group: networking.cloud.tencent.com + names: + kind: DedicatedCLBListener + listKind: DedicatedCLBListenerList + plural: dedicatedclblisteners + singular: dedicatedclblistener + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: CLB ID + jsonPath: .spec.lbId + name: LbId + type: string + - description: Port of CLB Listener + jsonPath: .spec.lbPort + name: LbPort + type: integer + - description: Pod name of target pod + jsonPath: .spec.targetPod.podName + name: Pod + type: string + - description: State of the dedicated clb listener + jsonPath: .status.state + name: State + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: DedicatedCLBListener is the Schema for the dedicatedclblisteners + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener + properties: + extensiveParameters: + description: 创建监听器的参数，JSON 格式，详细参数请参考 CreateListener 接口：https://cloud.tencent.com/document/api/214/30693 + type: string + lbId: + description: CLB 实例的 ID。 + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + lbPort: + description: CLB 监听器的端口号。 + format: int64 + type: integer + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + lbRegion: + description: CLB 所在地域，不填则使用 TKE 集群所在的地域。 + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + protocol: + description: CLB 监听器的协议。 + enum: + - TCP + - UDP + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + targetPod: + description: CLB 监听器绑定的目标 Pod。 + properties: + podName: + description: Pod 的名称。 + type: string + targetPort: + description: Pod 监听的端口。 + format: int64 + type: integer + required: + - podName + - targetPort + type: object + required: + - lbId + - lbPort + - protocol + type: object + status: + description: DedicatedCLBListenerStatus defines the observed state of + DedicatedCLBListener + properties: + address: + description: CLB 监听器的外部地址。 + type: string + listenerId: + description: CLB 监听器的 ID。 + type: string + message: + description: 记录 CLB 监听器的失败信息。 + type: string + state: + description: CLB 监听器的状态。 + enum: + - Bound + - Available + - Pending + - Failed + - Deleting + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclbservices.yaml b/incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclbservices.yaml new file mode 100644 index 00000000..30d66a73 --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/networking.cloud.tencent.com_dedicatedclbservices.yaml @@ -0,0 +1,152 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "tke-extend-network-controller.fullname" . }}-serving-cert + name: dedicatedclbservices.networking.cloud.tencent.com +spec: + group: networking.cloud.tencent.com + names: + kind: DedicatedCLBService + listKind: DedicatedCLBServiceList + plural: dedicatedclbservices + singular: dedicatedclbservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: DedicatedCLBService is the Schema for the dedicatedclbservices + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: DedicatedCLBServiceSpec defines the desired state of DedicatedCLBService + properties: + existedLbIds: + description: 复用的已有的 CLB ID，可动态追加。 + items: + type: string + type: array + lbAutoCreate: + description: 启用自动创建 CLB 的功能。 + properties: + enable: + description: 是否启用自动创建 CLB 的功能，如果启用，当 CLB 不足时，会自动创建新的 CLB。 + type: boolean + extensiveParameters: + description: 创建 CLB 时的参数，JSON 格式，详细参数请参考 CreateLoadBalancer 接口：https://cloud.tencent.com/document/api/214/30692 + type: string + type: object + lbRegion: + description: CLB 所在地域，不填则使用 TKE 集群所在的地域。 + type: string + listenerExtensiveParameters: + description: 创建监听器的参数，JSON 格式，详细参数请参考 CreateListener 接口：https://cloud.tencent.com/document/api/214/30693 + type: string + maxPod: + description: 限制单个 CLB 的 Pod/监听器 的最大数量。 + format: int64 + type: integer + maxPort: + default: 50000 + description: CLB 端口范围的最大端口号。 + format: int64 + type: integer + minPort: + default: 500 + description: CLB 端口范围的最小端口号。 + format: int64 + type: integer + ports: + description: Pod 监听的端口。 + items: + properties: + addressPodAnnotation: + description: Pod 外部地址的注解，如果设置，Pod 被映射的外部 CLB 地址将会被自动写到 Pod 的该注解中，Pod + 内部可通过 Downward API 感知到自身的外部地址。 + type: string + protocol: + description: 端口协议，支持 TCP、UDP。 + type: string + targetPort: + description: 目标端口。 + format: int64 + type: integer + required: + - protocol + - targetPort + type: object + type: array + selector: + additionalProperties: + type: string + description: Pod 的标签选择器，被选中的 Pod 会被绑定到 CLB 监听器下。 + type: object + vpcId: + description: CLB 所在 VPC ID，不填则使用 TKE 集群所在的 VPC 的 ID。 + type: string + required: + - ports + - selector + type: object + status: + description: DedicatedCLBServiceStatus defines the observed state of DedicatedCLBService + properties: + allocatableLb: + description: 可分配端口的 CLB 列表 + items: + properties: + autoCreate: + description: 是否是自动创建的 CLB。如果是，删除 DedicatedCLBService 时，CLB 也会被清理。 + type: boolean + currentPort: + description: CLB 当前已被分配的端口。 + format: int64 + type: integer + lbId: + description: CLB 实例的 ID。 + type: string + required: + - autoCreate + - lbId + type: object + type: array + allocatedLb: + description: 已分配完端口的 CLB 列表 + items: + properties: + autoCreate: + description: 是否是自动创建的 CLB。如果是，删除 DedicatedCLBService 时，CLB 也会被清理。 + type: boolean + lbId: + description: CLB 实例的 ID。 + type: string + required: + - autoCreate + - lbId + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} + diff --git a/incubator/tke-extend-network-controller/templates/role.yaml b/incubator/tke-extend-network-controller/templates/role.yaml new file mode 100644 index 00000000..8771fc4a --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/role.yaml @@ -0,0 +1,186 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }}-leader-election-role + namespace: {{ .Release.Namespace | quote }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "tke-extend-network-controller.fullname" . }}-metrics-auth-role + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "tke-extend-network-controller.fullname" . }}-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "tke-extend-network-controller.fullname" . }}-role + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatedclblisteners + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatedclblisteners/finalizers + verbs: + - update +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatedclblisteners/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatedclbservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatedclbservices/finalizers + verbs: + - update +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatedclbservices/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatednatgwservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatednatgwservices/finalizers + verbs: + - update +- apiGroups: + - networking.cloud.tencent.com + resources: + - dedicatednatgwservices/status + verbs: + - get + - patch + - update + diff --git a/incubator/tke-extend-network-controller/templates/rolebinding.yaml b/incubator/tke-extend-network-controller/templates/rolebinding.yaml new file mode 100644 index 00000000..06fab407 --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/rolebinding.yaml @@ -0,0 +1,45 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }}-leader-election-rolebinding + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "tke-extend-network-controller.fullname" . }}-leader-election-role +subjects: +- kind: ServiceAccount + name: {{ include "tke-extend-network-controller.fullname" . }} + namespace: {{ .Release.Namespace | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "tke-extend-network-controller.fullname" . }}-metrics-auth-rolebinding + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "tke-extend-network-controller.fullname" . }}-metrics-auth-role +subjects: +- kind: ServiceAccount + name: {{ include "tke-extend-network-controller.fullname" . }} + namespace: {{ .Release.Namespace | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }}-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "tke-extend-network-controller.fullname" . }}-role +subjects: +- kind: ServiceAccount + name: {{ include "tke-extend-network-controller.fullname" . }} + namespace: {{ .Release.Namespace | quote }} diff --git a/incubator/tke-extend-network-controller/templates/secret-env.yaml b/incubator/tke-extend-network-controller/templates/secret-env.yaml new file mode 100644 index 00000000..3c77945f --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/secret-env.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "tke-extend-network-controller.fullname" . }}-env + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- with .Values.region }} + REGION: '{{ . }}' + {{- end }} + SECRET_ID: '{{ required "A valid .secretID required!" .Values.secretID }}' + SECRET_KEY: '{{ required "A valid .secretKey required!" .Values.secretKey }}' + VPCID: '{{ required "A valid .vpcID required!" .Values.vpcID }}' + CLUSTER_ID: '{{ required "A valid .clusterID required!" .Values.clusterID }}' diff --git a/incubator/tke-extend-network-controller/templates/service.yaml b/incubator/tke-extend-network-controller/templates/service.yaml new file mode 100644 index 00000000..1dcd2ca2 --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: tke-extend-network-controller + control-plane: tke-extend-network-controller + name: {{ include "tke-extend-network-controller.fullname" . }} +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + {{- include "tke-extend-network-controller.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: tke-extend-network-controller + name: {{ include "tke-extend-network-controller.fullname" . }}-webhook-service +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + {{- include "tke-extend-network-controller.selectorLabels" . | nindent 4 }} + diff --git a/incubator/tke-extend-network-controller/templates/serviceaccount.yaml b/incubator/tke-extend-network-controller/templates/serviceaccount.yaml new file mode 100644 index 00000000..72d63a54 --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + diff --git a/incubator/tke-extend-network-controller/templates/webhook.yaml b/incubator/tke-extend-network-controller/templates/webhook.yaml new file mode 100644 index 00000000..afbefa2f --- /dev/null +++ b/incubator/tke-extend-network-controller/templates/webhook.yaml @@ -0,0 +1,62 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "tke-extend-network-controller.fullname" . }}-serving-cert + labels: + app.kubernetes.io/component: webhook + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }}-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ include "tke-extend-network-controller.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace | quote }} + path: /mutate-networking-cloud-tencent-com-v1alpha1-dedicatedclblistener + failurePolicy: Fail + name: mdedicatedclblistener.kb.io + rules: + - apiGroups: + - networking.cloud.tencent.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - dedicatedclblisteners + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "tke-extend-network-controller.fullname" . }}-serving-cert + labels: + app.kubernetes.io/component: webhook + {{- include "tke-extend-network-controller.labels" . | nindent 4 }} + name: {{ include "tke-extend-network-controller.fullname" . }}-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ include "tke-extend-network-controller.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace | quote }} + path: /validate-networking-cloud-tencent-com-v1alpha1-dedicatedclblistener + failurePolicy: Fail + name: vdedicatedclblistener.kb.io + rules: + - apiGroups: + - networking.cloud.tencent.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - dedicatedclblisteners + sideEffects: None + diff --git a/incubator/tke-extend-network-controller/values.yaml b/incubator/tke-extend-network-controller/values.yaml new file mode 100644 index 00000000..34db33dc --- /dev/null +++ b/incubator/tke-extend-network-controller/values.yaml @@ -0,0 +1,73 @@ +# Default values for tke-extend-network-controller. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Replica count of the controller pod +replicaCount: 1 + +# -- Region of the current TKE Cluster, optional. +region: "" +# -- VPC ID of the current TKE Cluster. +vpcID: "" +# -- Secret ID of the Tencent Cloud Account. +secretID: "" +# -- Secret Key of the Tencent Cloud Account. +secretKey: "" +# -- Cluster ID of the current TKE Cluster. +clusterID: "" +# -- Worker count of the controller +workerCount: 3 + +# -- Logging otpions of the controller +log: + # -- Log level of the controller, be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity + level: "info" + # -- Log format of the controller, be one of 'json' or 'console' + encoder: "json" + +# -- Image of the controller +image: + # -- Image repository of the controller + repository: ccr.ccs.tencentyun.com/tke-market/tke-extend-network-controller + # -- ImagePullPolicy of the controller + pullPolicy: IfNotPresent + # -- Overrides the image tag whose default is the chart appVersion. + tag: "" + +# -- ImagePullSecrets of the controller +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + +readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + +nodeSelector: {} + +tolerations: [] + +affinity: {} From fb8c34aad340b2d2dde87731297048a4139f9e95 Mon Sep 17 00:00:00 2001 From: rockerchen Date: Thu, 19 Sep 2024 10:05:12 +0800 Subject: [PATCH 2/2] update readme for tke-extend-network-controller --- .../tke-extend-network-controller/README.md | 158 +++++++++++++++++- 1 file changed, 154 insertions(+), 4 deletions(-) diff --git a/incubator/tke-extend-network-controller/README.md b/incubator/tke-extend-network-controller/README.md index 2f032c9d..3d7c032b 100644 --- a/incubator/tke-extend-network-controller/README.md +++ b/incubator/tke-extend-network-controller/README.md @@ -2,9 +2,161 @@ ![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square) -A Network Controller for TKE +针对 TKE 集群一些特殊场景的的网络控制器。 -**Homepage:** +## 支持房间类场景 + +目前主要支持会议、游戏战斗服等房间类场景的网络，即要求每个 Pod 都需要独立的公网地址，TKE 集群默认只支持 EIP 方案，但 EIP 资源有限，有申请的数量限制和每日申请的次数限制（参考 [EIP 配额限制](https://cloud.tencent.com/document/product/1199/41648#eip-.E9.85.8D.E9.A2.9D.E9.99.90.E5.88.B6)），稍微上点规模，或频繁扩缩容更换EIP，可能很容易触达限制导致 EIP 分配失败；而如果保留 EIP，在 EIP 没被绑定前，又会收取额外的闲置费。 + +> TKE Pod 绑定 EIP 参考 [Pod 绑 EIP](https://imroc.cc/tke/networking/pod-eip)。 +> +> 关于 EIP 与 CLB 映射两种方案的详细对比参考 [TKE 游戏方案：房间类游戏网络接入](https://imroc.cc/tke/game/room-networking)。 + +如果不用 EIP，也可通过安装此插件来实现为每个 Pod 的指定端口都分配一个独立的公网地址映射 (公网 `IP:Port` 到内网 Pod `IP:Port` 的映射)。 + +## 前提条件 + +安装 `tke-extend-network-controller` 前请确保满足以下前提条件： +1. 确保腾讯云账号是带宽上移账号，参考 [账户类型说明](https://cloud.tencent.com/document/product/1199/49090) 进行判断或升级账号类型（如果账号创建的时间很早，有可能是传统账号）。 +2. 创建了 [TKE](https://cloud.tencent.com/product/tke) 集群，且集群版本大于等于 1.26。 +3. 集群中安装了 [cert-manager](https://cert-manager.io/docs/installation/) (webhook 依赖证书)。 +4. 本地安装了 [helm](https://helm.sh) 命令，且能通过 helm 命令操作 TKE 集群（参考[本地 Helm 客户端连接集群](https://cloud.tencent.com/document/product/457/32731)）。 +5. 需要一个腾讯云子账号的访问密钥(SecretID、SecretKey)，参考[子账号访问密钥管理](https://cloud.tencent.com/document/product/598/37140)，要求账号至少具有以下权限： + ```json + { + "version": "2.0", + "statement": [ + { + "effect": "allow", + "action": [ + "clb:DescribeLoadBalancerBackends", + "clb:DescribeLoadBalancerListeners", + "clb:DescribeLoadBalancers", + "clb:CreateLoadBalancer", + "clb:DescribeTargets", + "clb:DeleteLoadBalancer", + "clb:DeleteLoadBalancerListeners", + "clb:BatchDeregisterTargets", + "clb:BatchRegisterTargets", + "clb:DeregisterTargets", + "clb:CreateLoadBalancerListeners", + "clb:CreateListener", + "clb:RegisterTargets", + "clb:DeleteLoadBalancers", + "clb:DescribeLoadBalancersDetail" + ], + "resource": [ + "*" + ] + } + ] + } + ``` + +## 配置 values + +以下是必要配置： + +```yaml +vpcID: "" # TKE 集群所在 VPC ID (vpc-xxx) +clusterID: "" # TKE 集群 ID (cls-xxx) +secretID: "" # 腾讯云子账号的 SecretID +secretKey: "" # 腾讯云子账号的 SecretKey +``` + +完整配置参考 [Values](#values)。 + +## 使用 CLB 为 Pod 分配公网地址映射 + +下面介绍如何为 Pod 分配独立的 CLB 公网地址映射。 + +### 创建 DedicatedCLBService + +为应用创建 `DedicatedCLBService`: +1. `selector` 选中目标应用 Pod 的 labels。 +2. `existedLbIds` 传入用于为 Pod 分配公网映射的 CLB 实例 ID 列表，可动态追加。 +3. `minPort` 和 `maxPort` 为 CLB 自动创建监听器的端口范围，每个端口只绑定一个 Pod。 +4. `maxPod` 用于限制最大 Pod/监听器 数量。 +5. `ports` 为 Pod 监听的端口列表，通常一个房间进程只监听一个端口。其中 `addressPodAnnotation` 用于 CLB 绑定 Pod 后，自动将其 CLB 外部映射地址自动注入到指定的 pod annotation 中，可结合 Kubernetes 的 Downward API 将外部地址挂载进容器内，以便让应用能够感知到自身的公网地址。 + +```yaml +apiVersion: networking.cloud.tencent.com/v1alpha1 +kind: DedicatedCLBService +metadata: + namespace: demo + name: gameserver +spec: + lbRegion: ap-chengdu # 可选，CLB 所在地域，默认为集群所在地域 + minPort: 500 # 可选，在 CLB 自动创建监听器，每个 Pod 占用一个端口，默认端口号范围在 500-50000 + maxPort: 50000 + maxPod: 50 # 可选，限制最大 Pod/监听器 数量。 + selector: + app: gameserver + ports: + - protocol: TCP # 端口监听的协议（TCP/UDP） + targetPort: 9000 # 容器监听的端口 (游戏战斗服、会议等进程监听的端口) + addressPodAnnotation: networking.cloud.tencent.com/external-address # 可选，将外部地址自动注入到指定的 pod annotation 中 + listenerExtensiveParameters: | # 可选，指定创建监听器时的参数(JSON 格式)，完整参考 CreateListener 接口： https://cloud.tencent.com/document/api/214/30693 （由于是一个监听器只挂一个 Pod，通常不需要自定义监听器配置，因为健康检查、调度算法这些配置，对于只有一个 RS 的监听器没有意义） + { + "DeregisterTargetRst": true + } + existedLbIds: # 复用已有的 CLB 实例，指定 CLB 实例 ID 的列表 + - lb-xxx + - lb-yyy + - lb-zzz + lbAutoCreate: + enable: true # 当 CLB 不足时，自动创建 CLB + extensiveParameters: | # 购买 CLB 时的参数(JSON 字符串格式)：按流量计费，超强型4实例规格，带宽上限 60 Gbps （完整参数列表参考 CreateLoadBalancer 接口 https://cloud.tencent.com/document/api/214/30692） + { + "InternetAccessible": { + "InternetChargeType": "TRAFFIC_POSTPAID_BY_HOUR", + "InternetMaxBandwidthOut": 61440 + }, + "SlaType": "clb.c4.xlarge" + } +``` + +### 将 CLB 映射的外部地址注入到 Pod 注解中 + +部署游戏服、会议等应用时，利用 Kubernetes 的 [Downward API](https://kubernetes.io/zh-cn/docs/tasks/inject-data-application/environment-variable-expose-pod-information/) 能力将记录 Pod 外部地址的注解内容挂载到容器内： + +```yaml + spec: + containers: + - ... + volumeMounts: + - name: podinfo + mountPath: /etc/podinfo + volumes: + - name: podinfo + downwardAPI: + items: + - path: "address" + fieldRef: + fieldPath: metadata.annotations['networking.cloud.tencent.com/external-address'] +``` + +### 容器内进程获取自身的 CLB 外部映射地址 + +进程启动时可轮询指定文件（本例中文件路径为 `/etc/podinfo/address`），当文件内容为空说明此时 Pod 还未绑定到 CLB，当读取到内容时说明已经绑定成功，其内容为 Pod 的 CLB 外部映射地址，进程可拿到该地址做进一步处理，如游戏战斗服上报自身房间的外部地址给大厅服或匹配服。 + +文件内容的格式为 `Host:Port`，其中 `Host` 是 CLB 的 VIP 或域名，所以格式又细分成以下两种形式： +1. `IP:Port`: CLB 的 VIP 加对外的端口号，如 `1.1.1.1:567`，非域名化的 CLB 会使用该形式。 +2. `Domain:Port`，CLB 的域名加对外的端口号，如 `lb-6q0yyqhb-p01vqztldre7is89.clb.cd-tencentclb.work:567`，域名化的 CLB 会使用该形式。 + +### CLB 的域名化与非域名化 + +通常新创建的 CLB 一般是域名化的 CLB（参考[此公告](https://cloud.tencent.com/document/product/214/86947)），即 CLB 没有固定的 VIP，它的外部地址就是一个域名，VIP 根据域名动态解析出来。 + +如何确定 CLB 是域名化还是非域名化呢？在 CLB 实例的详情页就可以看出来： + +![](./images/clb-domain.png) ![](./images/clb-fixed-vip.png) + +### 注意事项 + +CLB 有一些[默认的限制](https://cloud.tencent.com/document/product/214/6187)，其中每个 CLB 的监听器数量限制为 50，即最多创建 50 个端口。这个限制只是个软性限制，如有需要，也可以通过 [提工单](https://console.cloud.tencent.com/workorder/category) 来调大。 + +配置 `DedicatedCLBService` 时，注意 `minPort` 和 `maxPort` 的范围，避免超出限制，也根据实际需要来选择合适的端口区间，比如单个 Pod 如果承载流量很大，端口范围可缩小点，限制绑定的 Pod 数量，避免单个 CLB 流量过大超出带宽上限；反之如果流量小，可扩大端口范围来绑定更多 Pod。 ## Values @@ -40,5 +192,3 @@ A Network Controller for TKE | vpcID | string | `""` | VPC ID of the current TKE Cluster. | | workerCount | int | `3` | Worker count of the controller | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2)