Skip to content

Commit

Permalink
policy: do not chain cilium when policy is not required
Browse files Browse the repository at this point in the history 
Signed-off-by: l1b0k <[email protected]>
  • Loading branch information
@l1b0k
l1b0k committed Nov 9, 2024
1 parent ade466c commit a478ad2
Show file tree
Hide file tree
Showing 8 changed files with 157 additions and 96 deletions.
189 changes: 99 additions & 90 deletions cmd/terway-cli/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ const (
type feature struct {
EBPF bool
EDT bool

EnableNetworkPolicy bool
}

var (
Expand All @@ -56,7 +58,6 @@ func init() {
var cniCmd = &cobra.Command{
Use: "cni",
SilenceUsage: true,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
err := processCNIConfig(cmd, args)
if err != nil {
Expand All @@ -78,12 +79,11 @@ func processCNIConfig(cmd *cobra.Command, args []string) error {
return fmt.Errorf("failed to set feature gates: %v", err)
}

err = processInput(args)
err = processInput()
if err != nil {
return fmt.Errorf("failed process input: %v", err)
}

// mount bpf fs if needed
cni, err := os.ReadFile(outPutPath)
if err != nil {
return err
Expand All @@ -92,43 +92,34 @@ func processCNIConfig(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
for _, plugin := range cniJSON.Path("plugins").Children() {
if plugin.Path("type").Data().(string) == "cilium-cni" {
err = mountHostBpf()
if err != nil {
return err
}
}
}

return storeRuntimeConfig(nodeCapabilitiesFile, cniJSON)
}

func processInput(files []string) error {
func processInput() error {
cm, err := getAllConfig(eniConfBasePath)
if err != nil {
return err
}

input := cm.cniConfig
if cm.cniConfigList != nil {
input = cm.cniConfigList
}

var configs [][]byte
for _, file := range files {
out, err := os.ReadFile(file)
if err != nil {
if os.IsNotExist(err) {
continue
}
return err
}
c, err := gabs.ParseJSON(out)
if err != nil {
return err
}
if c.Exists("plugins") {
for _, cc := range c.Path("plugins").Children() {
configs = append(configs, cc.Bytes())
}
} else {
configs = append(configs, out)
c, err := gabs.ParseJSON(input)
if err != nil {
return err
}
if c.Exists("plugins") {
for _, cc := range c.Path("plugins").Children() {
configs = append(configs, cc.Bytes())
}
break
} else {
configs = append(configs, input)
}

var err error
f := feature{}
f.EBPF = _checkKernelVersion(4, 19, 0)

Expand All @@ -138,6 +129,9 @@ func processInput(files []string) error {
return err
}
}

f.EnableNetworkPolicy = cm.enableNetworkPolicy

out, err := mergeConfigList(configs, &f)
if err != nil {
return err
Expand Down Expand Up @@ -211,65 +205,62 @@ func mergeConfigList(configs [][]byte, f *feature) (string, error) {
return "", fmt.Errorf("network_policy_provider type error")
}
}
if plugin.Exists("eniip_virtual_type") {
virtualType, ok := plugin.Path("eniip_virtual_type").Data().(string)
if !ok {
return "", fmt.Errorf("eniip_virtual_type not found")
}
if !ebpfSupport {
err = plugin.Delete("eniip_virtual_type")
if err != nil {
return "", err
}
} else {
requireIPvlan := false

switch strings.ToLower(virtualType) {
case dataPathVeth, dataPathDefault:
datapath = dataPathVeth
virtualType, ok := plugin.Path("eniip_virtual_type").Data().(string)
if !ok {
virtualType = dataPathVeth
}
if !ebpfSupport {
_ = plugin.Delete("eniip_virtual_type")
} else {
requireIPvlan := false

switch strings.ToLower(virtualType) {
case dataPathVeth, dataPathDefault:
datapath = dataPathVeth

// only for terway-eniip
if ebpfSupport && networkPolicyProvider == NetworkPolicyProviderEBPF {
requireEBPFChainer = true
// only for terway-eniip
if ebpfSupport && networkPolicyProvider == NetworkPolicyProviderEBPF {
allow, err := allowEBPFNetworkPolicy(f.EnableNetworkPolicy)
if err != nil {
return "", err
}
case dataPathIPvlan:
requireIPvlan = true
datapath = dataPathIPvlan

fallthrough
case dataPathV2:
requireEBPFChainer = true

if requireIPvlan && !_switchDataPathV2() {
fmt.Printf("keep ipvlan mode %v %v\n", requireIPvlan, !_switchDataPathV2())
_, err = plugin.Set(dataPathIPvlan, "eniip_virtual_type")
if err != nil {
return "", err
}
} else {
fmt.Printf("datapathv2 enabled\n")
_, err = plugin.Set(dataPathV2, "eniip_virtual_type")
if err != nil {
return "", err
}

datapath = dataPathV2
if allow {
requireEBPFChainer = true
}
}
case dataPathIPvlan:
requireIPvlan = true
datapath = dataPathIPvlan

fallthrough
case dataPathV2:
requireEBPFChainer = true

if edtSupport {
_, err = plugin.Set("edt", "bandwidth_mode")
} else {
_, err = plugin.Set("tc", "bandwidth_mode")
if requireIPvlan && !_switchDataPathV2() {
fmt.Printf("keep ipvlan mode %v %v\n", requireIPvlan, !_switchDataPathV2())
_, err = plugin.Set(dataPathIPvlan, "eniip_virtual_type")
if err != nil {
return "", err
}
} else {
fmt.Printf("datapathv2 enabled\n")
_, err = plugin.Set(dataPathV2, "eniip_virtual_type")
if err != nil {
return "", err
}

datapath = dataPathV2
}

if edtSupport {
_, err = plugin.Set("edt", "bandwidth_mode")
} else {
_, err = plugin.Set("tc", "bandwidth_mode")
}
if err != nil {
return "", err
}
}
} else {
datapath = dataPathVeth
if ebpfSupport && networkPolicyProvider == NetworkPolicyProviderEBPF {
requireEBPFChainer = true
}
}
}
Expand Down Expand Up @@ -315,20 +306,38 @@ func storeRuntimeConfig(filePath string, container *gabs.Container) error {
return err
}

hasCilium := false
// write back current runtime config
for _, plugin := range container.Path("plugins").Children() {
if plugin.Path("type").Data().(string) != pluginTypeTerway {
continue
}
if plugin.Exists("network_policy_provider") {
networkPolicyProvider := plugin.Path("network_policy_provider").Data().(string)
store.Set(nodecap.NodeCapabilityNetworkPolicyProvider, networkPolicyProvider)
pluginType, ok := plugin.Path("type").Data().(string)
if !ok {
return fmt.Errorf("type must be string")
}
if plugin.Exists("eniip_virtual_type") {
datapath := plugin.Path("eniip_virtual_type").Data().(string)
store.Set(nodecap.NodeCapabilityDataPath, datapath)
switch pluginType {
case pluginTypeCilium:
// mount bpf fs if needed

err = mountHostBpf()
if err != nil {
return err
}
hasCilium = true
case pluginTypeTerway:
if plugin.Exists("network_policy_provider") {
networkPolicyProvider := plugin.Path("network_policy_provider").Data().(string)
store.Set(nodecap.NodeCapabilityNetworkPolicyProvider, networkPolicyProvider)
}
if plugin.Exists("eniip_virtual_type") {
datapath := plugin.Path("eniip_virtual_type").Data().(string)
store.Set(nodecap.NodeCapabilityDataPath, datapath)
}
}
}
if hasCilium {
store.Set(nodecap.NodeCapabilityHasCiliumChainer, True)
} else {
store.Set(nodecap.NodeCapabilityHasCiliumChainer, False)
}

return store.Save()
}
33 changes: 33 additions & 0 deletions cmd/terway-cli/cni_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,39 @@ func switchDataPathV2() bool {
return errors.As(err, &netlink.LinkNotFoundError{})
}

// allowEBPFNetworkPolicy check in veth datapath
// policy
// false -> true:
// old node(has cilium already) keep old behave
// old node(do not has cilium) keep old behave
// new node ( based on user require).
// true -> false: keep cilium chain, but disable policy
func allowEBPFNetworkPolicy(require bool) (bool, error) {
store := nodecap.NewFileNodeCapabilities(nodeCapabilitiesFile)
if err := store.Load(); err != nil {
return false, err
}
switch store.Get(nodecap.NodeCapabilityHasCiliumChainer) {
case True:
fmt.Printf("has prev cilium chainer\n")
return true, nil
case False:
fmt.Printf("no prev cilium chainer\n")
return false, nil
}

_, err := netlink.LinkByName("cilium_net")
if err == nil {
fmt.Printf("link cilium_net exist\n")
return true, nil
}
if !errors.As(err, &netlink.LinkNotFoundError{}) {
return false, err
}

return require, nil
}

func checkKernelVersion(k, major, minor int) bool {
return kernel.CheckKernelVersion(k, major, minor)
}
4 changes: 4 additions & 0 deletions cmd/terway-cli/cni_unsupport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,7 @@ func switchDataPathV2() bool {
func checkKernelVersion(k, major, minor int) bool {
return false
}

func allowEBPFNetworkPolicy(enable bool) (bool, error) {
return enable, nil
}
7 changes: 6 additions & 1 deletion cmd/terway-cli/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ const (
)
const eniConfBasePath = "/etc/eni"

const (
True = "true"
False = "false"
)

type TerwayConfig struct {
enableNetworkPolicy bool
enableInClusterLB bool
Expand Down Expand Up @@ -71,7 +76,7 @@ func getAllConfig(base string) (*TerwayConfig, error) {
return nil, err
}
}
if string(r) == "true" {
if string(r) == True {
cfg.enableInClusterLB = true
}

Expand Down
4 changes: 2 additions & 2 deletions cmd/terway-cli/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,9 @@ func dualStack(cmd *cobra.Command, args []string) error {
val := ""
switch eniCfg.IPStack {
case "dual", "ipv6":
val = "true"
val = True
default:
val = "false"
val = False
}

err := store.Load()
Expand Down
12 changes: 10 additions & 2 deletions cmd/terway-cli/policy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type PolicyConfig struct {
HealthCheckPort string
IPv6 bool
InClusterLoadBalance bool
HasCiliumChainer bool
}

type CNIConfig struct {
Expand Down Expand Up @@ -61,7 +62,7 @@ func getPolicyConfig(capFilePath string) (*PolicyConfig, error) {
return nil, err
}

if store.Get(nodecap.NodeCapabilityIPv6) == ("true") {
if store.Get(nodecap.NodeCapabilityIPv6) == True {
cfg.IPv6 = true
}

Expand All @@ -70,6 +71,7 @@ func getPolicyConfig(capFilePath string) (*PolicyConfig, error) {
}
cfg.Datapath = store.Get(nodecap.NodeCapabilityDataPath)
cfg.PolicyProvider = store.Get(nodecap.NodeCapabilityNetworkPolicyProvider)
cfg.HasCiliumChainer = store.Get(nodecap.NodeCapabilityHasCiliumChainer) == True

cfg.HealthCheckPort = os.Getenv("FELIX_HEALTHPORT")
if cfg.HealthCheckPort == "" {
Expand Down Expand Up @@ -107,6 +109,9 @@ func initPolicy(cmd *cobra.Command, args []string) error {
}
return runSocat(cfg)
}
if !cfg.HasCiliumChainer {
return runSocat(cfg)
}
fmt.Printf("enable ebpf provider, run cilium")
fallthrough
case dataPathIPvlan, dataPathV2:
Expand Down Expand Up @@ -167,6 +172,10 @@ func runCalico(cfg *PolicyConfig) error {
}

func runCilium(cfg *PolicyConfig) error {
if !cfg.HasCiliumChainer {
return fmt.Errorf("no cilium chainer is installed")
}

extraArgs, err := parsePolicyConfig()
if err != nil {
return err
Expand All @@ -193,7 +202,6 @@ func runCilium(cfg *PolicyConfig) error {
"--enable-l7-proxy=false",
"--ipam=cluster-pool",
"--enable-runtime-device-detection=true",
"--enable-policy=" + fmt.Sprintf("%t", cfg.EnableNetworkPolicy),
"--agent-health-port=" + cfg.HealthCheckPort,
}
if cfg.EnableNetworkPolicy {
Expand Down
Loading

0 comments on commit a478ad2

Please sign in to comment.