Skip to content

Commit 93745fa

Browse files
huangchenzhaohuangchenzhao
committed
1121
1 parent 3f1f88b commit 93745fa

File tree

10 files changed

+333
-29
lines changed

10 files changed

+333
-29
lines changed

‎pkg/yurtadm/cmd/join/join.go

+47-3
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package join
2020
import (
2121
"fmt"
2222
"io"
23+
"os"
2324
"strings"
2425

2526
"github.com/pkg/errors"
@@ -35,6 +36,7 @@ import (
3536
"github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/apiclient"
3637
"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
3738
yurtphases "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/phases"
39+
"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
3840
yurtconstants "github.com/openyurtio/openyurt/pkg/yurtadm/constants"
3941
"github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
4042
yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
@@ -52,6 +54,8 @@ type joinOptions struct {
5254
organizations string
5355
pauseImage string
5456
yurthubImage string
57+
yurthubBinary string
58+
hostControlPlaneAddr string // hostControlPlaneAddr is the address (ip:port) of host kubernetes cluster that used for yurthub local mode.
5559
namespace string
5660
caCertHashes []string
5761
unsafeSkipCAVerification bool
@@ -124,7 +128,7 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
124128
)
125129
flagSet.StringVar(
126130
&joinOptions.nodeType, yurtconstants.NodeType, joinOptions.nodeType,
127-
"Sets the node is edge or cloud",
131+
"Sets the node is edge, cloud or local",
128132
)
129133
flagSet.StringVar(
130134
&joinOptions.nodeName, yurtconstants.NodeName, joinOptions.nodeName,
@@ -154,6 +158,14 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
154158
&joinOptions.yurthubImage, yurtconstants.YurtHubImage, joinOptions.yurthubImage,
155159
"Sets the image version of yurthub component",
156160
)
161+
flagSet.StringVar(
162+
&joinOptions.yurthubBinary, yurtconstants.YurtHubBinary, joinOptions.yurthubBinary,
163+
"Sets the binary path of yurthub, this is used for deploying local mode yurthub in systemd",
164+
)
165+
flagSet.StringVar(
166+
&joinOptions.hostControlPlaneAddr, yurtconstants.HostControlPlaneAddr, joinOptions.hostControlPlaneAddr,
167+
"Sets the address of hostControlPlaneAddr, which is the address (ip:port) of host kubernetes cluster that used for yurthub local mode",
168+
)
157169
flagSet.StringSliceVar(
158170
&joinOptions.caCertHashes, yurtconstants.TokenDiscoveryCAHash, joinOptions.caCertHashes,
159171
"For token-based discovery, validate that the root CA public key matches this hash (format: \"<type>:<value>\").",
@@ -227,6 +239,8 @@ type joinData struct {
227239
organizations string
228240
pauseImage string
229241
yurthubImage string
242+
yurthubBinary string
243+
hostControlPlaneAddr string
230244
yurthubTemplate string
231245
yurthubManifest string
232246
kubernetesVersion string
@@ -257,6 +271,25 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
257271
apiServerEndpoint = args[0]
258272
}
259273

274+
if opt.nodeType == constants.LocalNode {
275+
// in local mode, it is necessary to prepare yurthub binary file for deploying systemd yurthub.
276+
if len(opt.yurthubBinary) == 0 {
277+
return nil, errors.New("yurthub binary filepath is empty, so unable to run systemd yurthub in local mode.")
278+
}
279+
_, err := os.Stat(opt.yurthubBinary)
280+
if err != nil {
281+
if os.IsNotExist(err) {
282+
return nil, errors.New("yurthub binary file does not exist.")
283+
}
284+
return nil, errors.Wrapf(err, "stat yurthub binary file %s fail", opt.yurthubBinary)
285+
}
286+
287+
// in local mode, hostControlPlaneAddr is needed for systemd yurthub accessing host kubernetes cluster.
288+
if len(opt.hostControlPlaneAddr) == 0 {
289+
return nil, errors.New("host control plane address is empty, so unable to run systemd yurthub in local mode.")
290+
}
291+
}
292+
260293
if len(opt.token) == 0 {
261294
return nil, errors.New("join token is empty, so unable to bootstrap worker node.")
262295
}
@@ -265,8 +298,8 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
265298
return nil, errors.Errorf("the bootstrap token %s was not of the form %s", opt.token, yurtconstants.BootstrapTokenPattern)
266299
}
267300

268-
if opt.nodeType != yurtconstants.EdgeNode && opt.nodeType != yurtconstants.CloudNode {
269-
return nil, errors.Errorf("node type(%s) is invalid, only \"edge and cloud\" are supported", opt.nodeType)
301+
if opt.nodeType != yurtconstants.EdgeNode && opt.nodeType != yurtconstants.CloudNode && opt.nodeType != yurtconstants.LocalNode {
302+
return nil, errors.Errorf("node type(%s) is invalid, only \"edge, cloud and local\" are supported", opt.nodeType)
270303
}
271304

272305
if opt.unsafeSkipCAVerification && len(opt.caCertHashes) != 0 {
@@ -298,6 +331,8 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
298331
ignorePreflightErrors: ignoreErrors,
299332
pauseImage: opt.pauseImage,
300333
yurthubImage: opt.yurthubImage,
334+
yurthubBinary: opt.yurthubBinary,
335+
hostControlPlaneAddr: opt.hostControlPlaneAddr,
301336
yurthubServer: opt.yurthubServer,
302337
caCertHashes: opt.caCertHashes,
303338
organizations: opt.organizations,
@@ -439,6 +474,15 @@ func (j *joinData) YurtHubImage() string {
439474
return j.yurthubImage
440475
}
441476

477+
// YurtHubBinary returns the YurtHub binary.
478+
func (j *joinData) YurtHubBinary() string {
479+
return j.yurthubBinary
480+
}
481+
482+
func (j *joinData) HostControlPlaneAddr() string {
483+
return j.hostControlPlaneAddr
484+
}
485+
442486
// YurtHubServer returns the YurtHub server addr.
443487
func (j *joinData) YurtHubServer() string {
444488
return j.yurthubServer

‎pkg/yurtadm/cmd/join/joindata/data.go

+2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ type YurtJoinData interface {
3636
JoinToken() string
3737
PauseImage() string
3838
YurtHubImage() string
39+
YurtHubBinary() string
40+
HostControlPlaneAddr() string
3941
YurtHubServer() string
4042
YurtHubTemplate() string
4143
YurtHubManifest() string

‎pkg/yurtadm/cmd/join/phases/postcheck.go

+18-8
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ import (
2020
"k8s.io/klog/v2"
2121

2222
"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
23+
"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
2324
"github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
25+
"github.com/openyurtio/openyurt/pkg/yurtadm/util/localnode"
2426
"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
2527
)
2628

@@ -33,15 +35,23 @@ func RunPostCheck(data joindata.YurtJoinData) error {
3335
klog.V(1).Infof("kubelet service is active")
3436

3537
klog.V(1).Infof("waiting hub agent ready.")
36-
if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil {
37-
return err
38-
}
39-
klog.V(1).Infof("hub agent is ready")
40-
41-
if err := yurthub.CleanHubBootstrapConfig(); err != nil {
42-
return err
38+
if data.NodeRegistration().WorkingMode == constants.LocalNode {
39+
// check systemd yurthub is ready or not
40+
if err := localnode.CheckYurthubStatus(); err != nil {
41+
return err
42+
}
43+
klog.V(1).Infof("systemd yurthub agent is ready")
44+
} else {
45+
if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil {
46+
return err
47+
}
48+
klog.V(1).Infof("staticpod yurthub agent is ready")
49+
50+
if err := yurthub.CleanHubBootstrapConfig(); err != nil {
51+
return err
52+
}
53+
klog.V(1).Infof("clean yurthub bootstrap config file success")
4354
}
44-
klog.V(1).Infof("clean yurthub bootstrap config file success")
4555

4656
return nil
4757
}

‎pkg/yurtadm/cmd/join/phases/prepare.go

+28-18
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,19 @@ import (
2626
"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
2727
"github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
2828
yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
29+
"github.com/openyurtio/openyurt/pkg/yurtadm/util/localnode"
2930
"github.com/openyurtio/openyurt/pkg/yurtadm/util/system"
3031
"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
3132
)
3233

3334
// RunPrepare executes the node initialization process.
3435
func RunPrepare(data joindata.YurtJoinData) error {
3536
// cleanup at first
36-
staticPodsPath := filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)
37-
if err := os.RemoveAll(staticPodsPath); err != nil {
38-
klog.Warningf("remove %s: %v", staticPodsPath, err)
37+
if data.NodeRegistration().WorkingMode != constants.LocalNode {
38+
staticPodsPath := filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)
39+
if err := os.RemoveAll(staticPodsPath); err != nil {
40+
klog.Warningf("remove %s: %v", staticPodsPath, err)
41+
}
3942
}
4043

4144
if err := system.SetIpv4Forward(); err != nil {
@@ -65,23 +68,30 @@ func RunPrepare(data joindata.YurtJoinData) error {
6568
if err := yurtadmutil.SetKubeletUnitConfig(); err != nil {
6669
return err
6770
}
68-
if err := yurtadmutil.SetKubeletConfigForNode(); err != nil {
69-
return err
70-
}
71-
if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
72-
return err
73-
}
74-
if err := yurthub.AddYurthubStaticYaml(data, constants.StaticPodPath); err != nil {
75-
return err
76-
}
77-
if len(data.StaticPodTemplateList()) != 0 {
78-
// deploy user specified static pods
79-
if err := edgenode.DeployStaticYaml(data.StaticPodManifestList(), data.StaticPodTemplateList(), constants.StaticPodPath); err != nil {
71+
if data.NodeRegistration().WorkingMode == constants.LocalNode {
72+
// deploy systemd yurthub in local mode
73+
if err := localnode.DeployYurthubInSystemd(data.HostControlPlaneAddr(), data.ServerAddr(), data.YurtHubBinary()); err != nil {
74+
return err
75+
}
76+
} else {
77+
if err := yurtadmutil.SetKubeletConfigForNode(); err != nil {
78+
return err
79+
}
80+
if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
81+
return err
82+
}
83+
if err := yurthub.AddYurthubStaticYaml(data, constants.StaticPodPath); err != nil {
84+
return err
85+
}
86+
if len(data.StaticPodTemplateList()) != 0 {
87+
// deploy user specified static pods
88+
if err := edgenode.DeployStaticYaml(data.StaticPodManifestList(), data.StaticPodTemplateList(), constants.StaticPodPath); err != nil {
89+
return err
90+
}
91+
}
92+
if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
8093
return err
8194
}
82-
}
83-
if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
84-
return err
8595
}
8696
if data.CfgPath() == "" {
8797
if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil {

‎pkg/yurtadm/constants/constants.go

+29
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ const (
3535
PauseImagePath = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"
3636
DefaultCertificatesDir = "/etc/kubernetes/pki"
3737
DefaultDockerCRISocket = "/var/run/dockershim.sock"
38+
YurthubServiceFilepath = "/etc/systemd/system/yurthub.service"
39+
YurthubEnvironmentFilePath = "/etc/systemd/system/yurthub.default"
3840
YurthubYamlName = "yurthub.yaml"
3941
YurthubStaticPodManifest = "yurthub"
4042
YurthubNamespace = "kube-system"
@@ -72,6 +74,7 @@ const (
7274

7375
EdgeNode = "edge"
7476
CloudNode = "cloud"
77+
LocalNode = "local"
7578

7679
// CertificatesDir
7780
CertificatesDir = "cert-dir"
@@ -107,6 +110,10 @@ const (
107110
Namespace = "namespace"
108111
// YurtHubImage flag sets the yurthub image for worker node.
109112
YurtHubImage = "yurthub-image"
113+
// YurtHubBinary flag sets the yurthub Binary for worker node.
114+
YurtHubBinary = "yurthub-binary"
115+
// HostControlPlaneAddr flag sets the address of host kubernetes cluster
116+
HostControlPlaneAddr = "host-control-plane-addr"
110117
// YurtHubServerAddr flag set the address of yurthub server (not proxy server!)
111118
YurtHubServerAddr = "yurthub-server-addr"
112119
// ServerAddr flag set the address of kubernetes kube-apiserver
@@ -272,5 +279,27 @@ spec:
272279
hostNetwork: true
273280
priorityClassName: system-node-critical
274281
priority: 2000001000
282+
`
283+
284+
YurthubSyetmdServiceContent = `
285+
[Unit]
286+
Description=local mode yurthub is deployed in systemd
287+
Documentation=https://github.com/openyurtio/openyurt/pull/2124
288+
289+
[Service]
290+
EnvironmentFile=/etc/systemd/system/yurthub.default
291+
ExecStart=/usr/bin/yurthub --working-mode ${WORKINGMODE} --node-name ${NODENAME} --server-addr ${SERVERADDR} --host-control-plane-address ${HOSTCONTROLPLANEADDRESS}
292+
Restart=always
293+
StartLimitInterval=0
294+
RestartSec=10
295+
296+
[Install]
297+
WantedBy=multi-user.target`
298+
299+
YurthubSyetmdServiceEnvironmentFileContent = `
300+
WORKINGMODE=local
301+
NODENAME=testnode
302+
SERVERADDR=121.40.250.117:6443
303+
HOSTCONTROLPLANEADDRESS=121.40.250.117:6443
275304
`
276305
)

‎pkg/yurtadm/util/initsystem/initsystem.go

+3
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,7 @@ type InitSystem interface {
2727

2828
// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet)
2929
ServiceIsActive(service string) bool
30+
31+
// ServiceToStart tries to start a specific service
32+
ServiceStart(service string) error
3033
}

‎pkg/yurtadm/util/initsystem/initsystem_unix.go

+16
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ func (openrc OpenRCInitSystem) ServiceIsActive(service string) bool {
5252
return !strings.Contains(outStr, "stopped") && !strings.Contains(outStr, "does not exist")
5353
}
5454

55+
// ServiceStart tries to start a specific service
56+
func (openrc OpenRCInitSystem) ServiceStart(service string) error {
57+
args := []string{service, "start"}
58+
return exec.Command("rc-service", args...).Run()
59+
}
60+
5561
// SystemdInitSystem defines systemd
5662
type SystemdInitSystem struct{}
5763

@@ -94,6 +100,16 @@ func (sysd SystemdInitSystem) ServiceIsActive(service string) bool {
94100
return false
95101
}
96102

103+
// ServiceStart tries to start a specific service
104+
func (sysd SystemdInitSystem) ServiceStart(service string) error {
105+
// Before we try to start any service, make sure that systemd is ready
106+
if err := sysd.reloadSystemd(); err != nil {
107+
return err
108+
}
109+
args := []string{"start", service}
110+
return exec.Command("systemctl", args...).Run()
111+
}
112+
97113
// GetInitSystem returns an InitSystem for the current system, or nil
98114
// if we cannot detect a supported init system.
99115
// This indicates we will skip init system checks, not an error.

0 commit comments

Comments
 (0)