Skip to content

Commit 13e18e6

Browse files
huangchenzhaohuangchenzhao
committed
test
1 parent 9ba1d5e commit 13e18e6

File tree

11 files changed

+416
-26
lines changed

11 files changed

+416
-26
lines changed

‎cmd/yurthub/app/config/config.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,8 @@ func createClientAndSharedInformers(options *options.YurtHubOptions) (kubernetes
279279
var err error
280280
// If yurthub is in local mode, create kubeconfig for host control plane to prepare informerFactory.
281281
if util.WorkingMode(options.WorkingMode) == util.WorkingModeLocal {
282-
kubeConfig, err = clientcmd.BuildConfigFromFlags(fmt.Sprintf("http://%s", options.HostControlPlaneAddr), "")
282+
// kubeConfig, err = clientcmd.BuildConfigFromFlags(fmt.Sprintf("http://%s", options.HostControlPlaneAddr), "")
283+
kubeConfig, err = clientcmd.BuildConfigFromFlags("", "/root/ackkubeconfig")
283284
} else {
284285
kubeConfig, err = clientcmd.BuildConfigFromFlags(fmt.Sprintf("http://%s:%d", options.YurtHubProxyHost, options.YurtHubProxyPort), "")
285286
}

‎pkg/yurtadm/cmd/join/join.go

+80-3
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package join
2020
import (
2121
"fmt"
2222
"io"
23+
"os"
2324
"strings"
2425

2526
"github.com/pkg/errors"
@@ -38,6 +39,7 @@ import (
3839
yurtconstants "github.com/openyurtio/openyurt/pkg/yurtadm/constants"
3940
"github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
4041
yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
42+
"github.com/openyurtio/openyurt/pkg/yurtadm/util/localnode"
4143
"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
4244
"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util"
4345
)
@@ -52,6 +54,8 @@ type joinOptions struct {
5254
organizations string
5355
pauseImage string
5456
yurthubImage string
57+
yurthubBinary string
58+
hostControlPlaneAddr string // hostControlPlaneAddr is the address (ip:port) of host kubernetes cluster that used for yurthub local mode.
5559
namespace string
5660
caCertHashes []string
5761
unsafeSkipCAVerification bool
@@ -124,7 +128,7 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
124128
)
125129
flagSet.StringVar(
126130
&joinOptions.nodeType, yurtconstants.NodeType, joinOptions.nodeType,
127-
"Sets the node is edge or cloud",
131+
"Sets the node is edge, cloud or local",
128132
)
129133
flagSet.StringVar(
130134
&joinOptions.nodeName, yurtconstants.NodeName, joinOptions.nodeName,
@@ -154,6 +158,14 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
154158
&joinOptions.yurthubImage, yurtconstants.YurtHubImage, joinOptions.yurthubImage,
155159
"Sets the image version of yurthub component",
156160
)
161+
flagSet.StringVar(
162+
&joinOptions.yurthubBinary, yurtconstants.YurtHubBinary, joinOptions.yurthubBinary,
163+
"Sets the binary path of yurthub, this is used for deploying local mode yurthub in systemd",
164+
)
165+
flagSet.StringVar(
166+
&joinOptions.hostControlPlaneAddr, yurtconstants.HostControlPlaneAddr, joinOptions.hostControlPlaneAddr,
167+
"Sets the address of hostControlPlaneAddr, which is the address (ip:port) of host kubernetes cluster that used for yurthub local mode",
168+
)
157169
flagSet.StringSliceVar(
158170
&joinOptions.caCertHashes, yurtconstants.TokenDiscoveryCAHash, joinOptions.caCertHashes,
159171
"For token-based discovery, validate that the root CA public key matches this hash (format: \"<type>:<value>\").",
@@ -227,6 +239,9 @@ type joinData struct {
227239
organizations string
228240
pauseImage string
229241
yurthubImage string
242+
yurthubBinary string
243+
hostControlPlaneAddr string
244+
tenantApiServerEndpoints string
230245
yurthubTemplate string
231246
yurthubManifest string
232247
kubernetesVersion string
@@ -257,6 +272,25 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
257272
apiServerEndpoint = args[0]
258273
}
259274

275+
if opt.nodeType == yurtconstants.LocalNode {
276+
// in local mode, it is necessary to prepare yurthub binary file for deploying systemd yurthub.
277+
if len(opt.yurthubBinary) == 0 {
278+
return nil, errors.New("yurthub binary filepath is empty, so unable to run systemd yurthub in local mode.")
279+
}
280+
_, err := os.Stat(opt.yurthubBinary)
281+
if err != nil {
282+
if os.IsNotExist(err) {
283+
return nil, errors.New("yurthub binary file does not exist.")
284+
}
285+
return nil, errors.Wrapf(err, "stat yurthub binary file %s fail", opt.yurthubBinary)
286+
}
287+
288+
// in local mode, hostControlPlaneAddr is needed for systemd yurthub accessing host kubernetes cluster.
289+
if len(opt.hostControlPlaneAddr) == 0 {
290+
return nil, errors.New("host control plane address is empty, so unable to run systemd yurthub in local mode.")
291+
}
292+
}
293+
260294
if len(opt.token) == 0 {
261295
return nil, errors.New("join token is empty, so unable to bootstrap worker node.")
262296
}
@@ -265,8 +299,8 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
265299
return nil, errors.Errorf("the bootstrap token %s was not of the form %s", opt.token, yurtconstants.BootstrapTokenPattern)
266300
}
267301

268-
if opt.nodeType != yurtconstants.EdgeNode && opt.nodeType != yurtconstants.CloudNode {
269-
return nil, errors.Errorf("node type(%s) is invalid, only \"edge and cloud\" are supported", opt.nodeType)
302+
if opt.nodeType != yurtconstants.EdgeNode && opt.nodeType != yurtconstants.CloudNode && opt.nodeType != yurtconstants.LocalNode {
303+
return nil, errors.Errorf("node type(%s) is invalid, only \"edge, cloud and local\" are supported", opt.nodeType)
270304
}
271305

272306
if opt.unsafeSkipCAVerification && len(opt.caCertHashes) != 0 {
@@ -298,6 +332,8 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
298332
ignorePreflightErrors: ignoreErrors,
299333
pauseImage: opt.pauseImage,
300334
yurthubImage: opt.yurthubImage,
335+
yurthubBinary: opt.yurthubBinary,
336+
hostControlPlaneAddr: opt.hostControlPlaneAddr,
301337
yurthubServer: opt.yurthubServer,
302338
caCertHashes: opt.caCertHashes,
303339
organizations: opt.organizations,
@@ -327,13 +363,37 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
327363
}
328364
}
329365

366+
// if the node type is local, before get tls bootstrap config, we need to deploy systemd yurthub for maintaining iptables rules
367+
if opt.nodeType == yurtconstants.LocalNode {
368+
// deploy systemd yurthub
369+
if err := localnode.DeployYurthubInSystemd(data.HostControlPlaneAddr(), data.ServerAddr(), data.YurtHubBinary(), data.NodeRegistration().Name); err != nil {
370+
klog.Errorf("could not deploy local yurthub in systemd, %v", err)
371+
return nil, err
372+
}
373+
374+
// check systemd yurthub is ready or not
375+
if err := localnode.CheckYurthubStatus(); err != nil {
376+
return nil, err
377+
}
378+
klog.V(1).Infof("systemd yurthub agent is ready")
379+
380+
tenantApiServerEndpoints, err := localnode.GetTenantApiServerEndpoints()
381+
if err != nil {
382+
klog.Errorf("could not get tenantApiServerEndpoints, %v", err)
383+
return nil, err
384+
}
385+
data.tenantApiServerEndpoints = tenantApiServerEndpoints
386+
klog.V(1).Infof("get tenantApiServerEndpoints: %s", tenantApiServerEndpoints)
387+
}
388+
330389
// get tls bootstrap config
331390
cfg, err := yurtadmutil.RetrieveBootstrapConfig(data)
332391
if err != nil {
333392
klog.Errorf("could not retrieve bootstrap config, %v", err)
334393
return nil, err
335394
}
336395
data.tlsBootstrapCfg = cfg
396+
klog.Infof("RetrieveBootstrapConfig: %#+v", *cfg)
337397

338398
// get kubernetes version
339399
client, err := kubeconfigutil.ToClientSet(cfg)
@@ -342,6 +402,7 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
342402
return nil, err
343403
}
344404
data.clientSet = client
405+
klog.Infof("ToClientSet: %#+v", *client)
345406

346407
k8sVersion, err := yurtadmutil.GetKubernetesVersionFromCluster(client)
347408
if err != nil {
@@ -350,6 +411,9 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
350411
}
351412
data.kubernetesVersion = k8sVersion
352413

414+
// test for get k8s version
415+
klog.Infof("GetKubernetesVersionFromCluster: %s", k8sVersion)
416+
353417
// check whether specified nodePool exists
354418
if len(opt.nodePoolName) != 0 {
355419
np, err := apiclient.GetNodePoolInfoWithRetry(cfg, opt.nodePoolName)
@@ -439,6 +503,19 @@ func (j *joinData) YurtHubImage() string {
439503
return j.yurthubImage
440504
}
441505

506+
// YurtHubBinary returns the YurtHub binary.
507+
func (j *joinData) YurtHubBinary() string {
508+
return j.yurthubBinary
509+
}
510+
511+
func (j *joinData) HostControlPlaneAddr() string {
512+
return j.hostControlPlaneAddr
513+
}
514+
515+
func (j *joinData) TenantApiServerEndpoints() string {
516+
return j.tenantApiServerEndpoints
517+
}
518+
442519
// YurtHubServer returns the YurtHub server addr.
443520
func (j *joinData) YurtHubServer() string {
444521
return j.yurthubServer

‎pkg/yurtadm/cmd/join/joindata/data.go

+3
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ type YurtJoinData interface {
3636
JoinToken() string
3737
PauseImage() string
3838
YurtHubImage() string
39+
YurtHubBinary() string
40+
HostControlPlaneAddr() string
41+
TenantApiServerEndpoints() string
3942
YurtHubServer() string
4043
YurtHubTemplate() string
4144
YurtHubManifest() string

‎pkg/yurtadm/cmd/join/phases/postcheck.go

+12-7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"k8s.io/klog/v2"
2121

2222
"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
23+
"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
2324
"github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
2425
"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
2526
)
@@ -33,15 +34,19 @@ func RunPostCheck(data joindata.YurtJoinData) error {
3334
klog.V(1).Infof("kubelet service is active")
3435

3536
klog.V(1).Infof("waiting hub agent ready.")
36-
if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil {
37-
return err
38-
}
39-
klog.V(1).Infof("hub agent is ready")
4037

41-
if err := yurthub.CleanHubBootstrapConfig(); err != nil {
42-
return err
38+
// check staticpod yurthub for edge node and cloud node
39+
if data.NodeRegistration().WorkingMode != constants.LocalNode {
40+
if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil {
41+
return err
42+
}
43+
klog.V(1).Infof("staticpod yurthub agent is ready")
44+
45+
if err := yurthub.CleanHubBootstrapConfig(); err != nil {
46+
return err
47+
}
48+
klog.V(1).Infof("clean yurthub bootstrap config file success")
4349
}
44-
klog.V(1).Infof("clean yurthub bootstrap config file success")
4550

4651
return nil
4752
}

‎pkg/yurtadm/cmd/join/phases/prepare.go

+17-15
Original file line numberDiff line numberDiff line change
@@ -65,23 +65,25 @@ func RunPrepare(data joindata.YurtJoinData) error {
6565
if err := yurtadmutil.SetKubeletUnitConfig(); err != nil {
6666
return err
6767
}
68-
if err := yurtadmutil.SetKubeletConfigForNode(); err != nil {
69-
return err
70-
}
71-
if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
72-
return err
73-
}
74-
if err := yurthub.AddYurthubStaticYaml(data, constants.StaticPodPath); err != nil {
75-
return err
76-
}
77-
if len(data.StaticPodTemplateList()) != 0 {
78-
// deploy user specified static pods
79-
if err := edgenode.DeployStaticYaml(data.StaticPodManifestList(), data.StaticPodTemplateList(), constants.StaticPodPath); err != nil {
68+
if data.NodeRegistration().WorkingMode != constants.LocalNode {
69+
if err := yurtadmutil.SetKubeletConfigForNode(); err != nil {
70+
return err
71+
}
72+
if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
73+
return err
74+
}
75+
if err := yurthub.AddYurthubStaticYaml(data, constants.StaticPodPath); err != nil {
76+
return err
77+
}
78+
if len(data.StaticPodTemplateList()) != 0 {
79+
// deploy user specified static pods
80+
if err := edgenode.DeployStaticYaml(data.StaticPodManifestList(), data.StaticPodTemplateList(), constants.StaticPodPath); err != nil {
81+
return err
82+
}
83+
}
84+
if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
8085
return err
8186
}
82-
}
83-
if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
84-
return err
8587
}
8688
if data.CfgPath() == "" {
8789
if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil {

‎pkg/yurtadm/constants/constants.go

+23
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ const (
3535
PauseImagePath = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"
3636
DefaultCertificatesDir = "/etc/kubernetes/pki"
3737
DefaultDockerCRISocket = "/var/run/dockershim.sock"
38+
YurthubServiceFilepath = "/etc/systemd/system/yurthub.service"
39+
YurthubEnvironmentFilePath = "/etc/systemd/system/yurthub.default"
3840
YurthubYamlName = "yurthub.yaml"
3941
YurthubStaticPodManifest = "yurthub"
4042
YurthubNamespace = "kube-system"
@@ -72,6 +74,7 @@ const (
7274

7375
EdgeNode = "edge"
7476
CloudNode = "cloud"
77+
LocalNode = "local"
7578

7679
// CertificatesDir
7780
CertificatesDir = "cert-dir"
@@ -107,6 +110,10 @@ const (
107110
Namespace = "namespace"
108111
// YurtHubImage flag sets the yurthub image for worker node.
109112
YurtHubImage = "yurthub-image"
113+
// YurtHubBinary flag sets the yurthub Binary for worker node.
114+
YurtHubBinary = "yurthub-binary"
115+
// HostControlPlaneAddr flag sets the address of host kubernetes cluster
116+
HostControlPlaneAddr = "host-control-plane-addr"
110117
// YurtHubServerAddr flag set the address of yurthub server (not proxy server!)
111118
YurtHubServerAddr = "yurthub-server-addr"
112119
// ServerAddr flag set the address of kubernetes kube-apiserver
@@ -272,5 +279,21 @@ spec:
272279
hostNetwork: true
273280
priorityClassName: system-node-critical
274281
priority: 2000001000
282+
`
283+
284+
YurthubSyetmdServiceContent = `
285+
[Unit]
286+
Description=local mode yurthub is deployed in systemd
287+
Documentation=https://github.com/openyurtio/openyurt/pull/2124
288+
289+
[Service]
290+
EnvironmentFile=/etc/systemd/system/yurthub.default
291+
ExecStart=/usr/bin/yurthub --working-mode ${WORKINGMODE} --node-name ${NODENAME} --server-addr ${SERVERADDR} --host-control-plane-address ${HOSTCONTROLPLANEADDRESS}
292+
Restart=always
293+
StartLimitInterval=0
294+
RestartSec=10
295+
296+
[Install]
297+
WantedBy=multi-user.target
275298
`
276299
)

‎pkg/yurtadm/util/initsystem/initsystem.go

+3
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,7 @@ type InitSystem interface {
2727

2828
// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet)
2929
ServiceIsActive(service string) bool
30+
31+
// ServiceToStart tries to start a specific service
32+
ServiceStart(service string) error
3033
}

‎pkg/yurtadm/util/initsystem/initsystem_unix.go

+16
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ func (openrc OpenRCInitSystem) ServiceIsActive(service string) bool {
5252
return !strings.Contains(outStr, "stopped") && !strings.Contains(outStr, "does not exist")
5353
}
5454

55+
// ServiceStart tries to start a specific service
56+
func (openrc OpenRCInitSystem) ServiceStart(service string) error {
57+
args := []string{service, "start"}
58+
return exec.Command("rc-service", args...).Run()
59+
}
60+
5561
// SystemdInitSystem defines systemd
5662
type SystemdInitSystem struct{}
5763

@@ -94,6 +100,16 @@ func (sysd SystemdInitSystem) ServiceIsActive(service string) bool {
94100
return false
95101
}
96102

103+
// ServiceStart tries to start a specific service
104+
func (sysd SystemdInitSystem) ServiceStart(service string) error {
105+
// Before we try to start any service, make sure that systemd is ready
106+
if err := sysd.reloadSystemd(); err != nil {
107+
return err
108+
}
109+
args := []string{"start", service}
110+
return exec.Command("systemctl", args...).Run()
111+
}
112+
97113
// GetInitSystem returns an InitSystem for the current system, or nil
98114
// if we cannot detect a supported init system.
99115
// This indicates we will skip init system checks, not an error.

0 commit comments

Comments
 (0)