Skip to content

Commit

Permalink
Refactor: Use interface to configure libbpf global data when loading …
Browse files Browse the repository at this point in the history 
…bpf programs. (#9663)

* Modify configuring libbpf global data

* Fix build

* Use Set method libbpf globals

* Refactor loading bpf programs to make use of a single API

* Fix fmt

* Address review comments
  • Loading branch information
@sridhartigera
sridhartigera authored Jan 7, 2025
1 parent 342d8ec commit 83e65f6
Show file tree
Hide file tree
Showing 9 changed files with 236 additions and 377 deletions.
69 changes: 67 additions & 2 deletions felix/bpf/bpf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ import (
"golang.org/x/sys/unix"

"github.com/projectcalico/calico/felix/bpf/bpfdefs"
"github.com/projectcalico/calico/felix/bpf/hook"
"github.com/projectcalico/calico/felix/bpf/libbpf"
"github.com/projectcalico/calico/felix/bpf/maps"
"github.com/projectcalico/calico/felix/bpf/utils"
"github.com/projectcalico/calico/felix/environment"
"github.com/projectcalico/calico/felix/labelindex"
Expand Down Expand Up @@ -2247,7 +2248,7 @@ func PolicyDebugJSONFileName(iface, polDir string, ipFamily proto.IPVersion) str
return path.Join(RuntimePolDir, fmt.Sprintf("%s_%s_v%d.json", iface, polDir, ipFamily))
}

func MapPinDir(typ int, name, iface string, h hook.Hook) string {
func MapPinDir() string {
PinBaseDir := path.Join(bpfdefs.DefaultBPFfsPath, "tc")
subDir := "globals"
return path.Join(PinBaseDir, subDir)
Expand Down Expand Up @@ -2349,3 +2350,67 @@ func IterPerCpuMapCmdOutput(output []byte, f func(k, v []byte)) error {
}
return nil
}

func LoadObject(file string, data libbpf.GlobalData, mapsToBePinned ...string) (*libbpf.Obj, error) {
obj, err := libbpf.OpenObject(file)
if err != nil {
return nil, err
}

success := false
defer func() {
if !success {
err := obj.Close()
if err != nil {
log.WithError(err).Error("Error closing BPF object.")
}
}
}()

for m, err := obj.FirstMap(); m != nil && err == nil; m, err = m.NextMap() {
// In case of global variables, libbpf creates an internal map <prog_name>.rodata
// The values are read only for the BPF programs, but can be set to a value from
// userspace before the program is loaded.
mapName := m.Name()
if m.IsMapInternal() {
if strings.HasPrefix(mapName, ".rodata") {
continue
}

if err := data.Set(m); err != nil {
return nil, fmt.Errorf("failed to configure %s: %w", file, err)
}
continue
}

if size := maps.Size(mapName); size != 0 {
if err := m.SetSize(size); err != nil {
return nil, fmt.Errorf("error resizing map %s: %w", mapName, err)
}
}

log.Debugf("Pinning map %s k %d v %d", mapName, m.KeySize(), m.ValueSize())
pinDir := MapPinDir()
// If mapsToBePinned is not specified, pin all the maps.
if len(mapsToBePinned) == 0 {
if err := m.SetPinPath(path.Join(pinDir, mapName)); err != nil {
return nil, fmt.Errorf("error pinning map %s k %d v %d: %w", mapName, m.KeySize(), m.ValueSize(), err)
}
} else {
for _, name := range mapsToBePinned {
if mapName == name {
if err := m.SetPinPath(path.Join(pinDir, mapName)); err != nil {
return nil, fmt.Errorf("error pinning map %s k %d v %d: %w", mapName, m.KeySize(), m.ValueSize(), err)
}
}
}
}
}

if err := obj.Load(); err != nil {
return nil, fmt.Errorf("error loading program: %w", err)
}

success = true
return obj, nil
}
91 changes: 12 additions & 79 deletions felix/bpf/conntrack/bpf_scanner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"github.com/projectcalico/calico/felix/bpf"
"github.com/projectcalico/calico/felix/bpf/bpfdefs"
"github.com/projectcalico/calico/felix/bpf/libbpf"
"github.com/projectcalico/calico/felix/bpf/maps"
)

type BPFLogLevel string
Expand Down Expand Up @@ -112,91 +111,25 @@ func (s *BPFProgLivenessScanner) ensureBPFExpiryProgram() (*libbpf.Obj, error) {
// needs a newer than co-re.
binaryToLoad := path.Join(bpfdefs.ObjectDir,
fmt.Sprintf("conntrack_cleanup_%s_co-re_v%d.o", s.logLevel, s.ipVersion))
obj, err := libbpf.OpenObject(binaryToLoad)
if err != nil {
return nil, fmt.Errorf("failed to load conntrack cleanup BPF program: %w", err)
}

success := false
defer func() {
if !success {
err := obj.Close()
if err != nil {
log.WithError(err).Error("Error closing BPF object.")
}
}
}()

ctMapParams := MapParams
if s.ipVersion == 6 {
ctMapParams = MapParamsV6
}
configuredGlobals := false
pinnedCTMap := false
var internalMaps []string
for m, err := obj.FirstMap(); m != nil && err == nil; m, err = m.NextMap() {
// In case of global variables, libbpf creates an internal map <prog_name>.rodata
// The values are read only for the BPF programs, but can be set to a value from
// userspace before the program is loaded.
mapName := m.Name()
if m.IsMapInternal() {
internalMaps = append(internalMaps, mapName)
if mapName != "conntrac.rodata" {
continue
}

err := libbpf.CTCleanupSetGlobals(
m,
s.timeouts.CreationGracePeriod,
s.timeouts.TCPPreEstablished,
s.timeouts.TCPEstablished,
s.timeouts.TCPFinsSeen,
s.timeouts.TCPResetSeen,
s.timeouts.UDPLastSeen,
s.timeouts.GenericIPLastSeen,
s.timeouts.ICMPLastSeen,
)
if err != nil {
return nil, fmt.Errorf("error setting global variables for map %s: %w", mapName, err)
}
configuredGlobals = true
continue
}

if size := maps.Size(mapName); size != 0 {
log.WithField("mapName", mapName).Info("Resizing map")
if err := m.SetSize(size); err != nil {
return nil, fmt.Errorf("error resizing map %s: %w", mapName, err)
}
}

if mapName == ctMapParams.VersionedName() {
log.Debugf("Pinning map %s k %d v %d", mapName, m.KeySize(), m.ValueSize())
pinDir := bpf.MapPinDir(m.Type(), mapName, "", 0)
if err := m.SetPinPath(path.Join(pinDir, mapName)); err != nil {
return nil, fmt.Errorf("error pinning map %s k %d v %d: %w", mapName, m.KeySize(), m.ValueSize(), err)
}
pinnedCTMap = true
}
}

if !configuredGlobals {
// Panic here because it indicates a coding error that we want to
// catch in testing.
log.WithField("maps", internalMaps).Panic("Bug: failed to find/set global variable map.")
}

if !pinnedCTMap {
// Panic here because it indicates a coding error that we want to
// catch in testing.
log.Panic("Bug: failed to find/pin conntrack map.")
}
ctCleanupData := &libbpf.CTCleanupGlobalData{
CreationGracePeriod: s.timeouts.CreationGracePeriod,
TCPPreEstablished: s.timeouts.TCPPreEstablished,
TCPEstablished: s.timeouts.TCPEstablished,
TCPFinsSeen: s.timeouts.TCPFinsSeen,
TCPResetSeen: s.timeouts.TCPResetSeen,
UDPLastSeen: s.timeouts.UDPLastSeen,
GenericIPLastSeen: s.timeouts.GenericIPLastSeen,
ICMPLastSeen: s.timeouts.ICMPLastSeen}

if err := obj.Load(); err != nil {
return nil, fmt.Errorf("error loading conntrack expiry program: %w", err)
obj, err := bpf.LoadObject(binaryToLoad, ctCleanupData, ctMapParams.VersionedName())
if err != nil {
return nil, fmt.Errorf("error loading %s: %w", binaryToLoad, err)
}

success = true
s.bpfExpiryProgram = obj
return s.bpfExpiryProgram, nil
}
Expand Down
116 changes: 49 additions & 67 deletions felix/bpf/libbpf/libbpf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,103 +384,85 @@ const (
GlobalsRedirectPeer uint32 = C.CALI_GLOBALS_REDIRECT_PEER
)

func CTCleanupSetGlobals(
m *Map,
CreationGracePeriod time.Duration,
TCPPreEstablished time.Duration,
TCPEstablished time.Duration,
TCPFinsSeen time.Duration,
TCPResetSeen time.Duration,
UDPLastSeen time.Duration,
GenericIPLastSeen time.Duration,
ICMPLastSeen time.Duration,
) error {
_, err := C.bpf_ct_cleanup_set_globals(
m.bpfMap,
C.uint64_t(CreationGracePeriod.Nanoseconds()),

C.uint64_t(TCPPreEstablished.Nanoseconds()),
C.uint64_t(TCPEstablished.Nanoseconds()),
C.uint64_t(TCPFinsSeen.Nanoseconds()),
C.uint64_t(TCPResetSeen.Nanoseconds()),

C.uint64_t(UDPLastSeen.Nanoseconds()),
C.uint64_t(GenericIPLastSeen.Nanoseconds()),
C.uint64_t(ICMPLastSeen.Nanoseconds()),
)
return err
}

func TcSetGlobals(
m *Map,
globalData *TcGlobalData,
) error {

cName := C.CString(globalData.IfaceName)
func (t *TcGlobalData) Set(m *Map) error {
cName := C.CString(t.IfaceName)
defer C.free(unsafe.Pointer(cName))

cJumps := make([]C.uint, len(globalData.Jumps))
cJumps := make([]C.uint, len(t.Jumps))

for i, v := range globalData.Jumps {
for i, v := range t.Jumps {
cJumps[i] = C.uint(v)
}

cJumpsV6 := make([]C.uint, len(globalData.JumpsV6))
cJumpsV6 := make([]C.uint, len(t.JumpsV6))

for i, v := range globalData.JumpsV6 {
for i, v := range t.JumpsV6 {
cJumpsV6[i] = C.uint(v)
}

_, err := C.bpf_tc_set_globals(m.bpfMap,
cName,
(*C.char)(unsafe.Pointer(&globalData.HostIPv4[0])),
(*C.char)(unsafe.Pointer(&globalData.IntfIPv4[0])),
(*C.char)(unsafe.Pointer(&globalData.HostIPv6[0])),
(*C.char)(unsafe.Pointer(&globalData.IntfIPv6[0])),
C.uint(globalData.ExtToSvcMark),
C.ushort(globalData.Tmtu),
C.ushort(globalData.VxlanPort),
C.ushort(globalData.PSNatStart),
C.ushort(globalData.PSNatLen),
(*C.char)(unsafe.Pointer(&globalData.HostTunnelIPv4[0])),
(*C.char)(unsafe.Pointer(&globalData.HostTunnelIPv6[0])),
C.uint(globalData.Flags),
C.ushort(globalData.WgPort),
C.ushort(globalData.Wg6Port),
C.ushort(globalData.Profiling),
C.uint(globalData.NatIn),
C.uint(globalData.NatOut),
C.uint(globalData.LogFilterJmp),
(*C.char)(unsafe.Pointer(&t.HostIPv4[0])),
(*C.char)(unsafe.Pointer(&t.IntfIPv4[0])),
(*C.char)(unsafe.Pointer(&t.HostIPv6[0])),
(*C.char)(unsafe.Pointer(&t.IntfIPv6[0])),
C.uint(t.ExtToSvcMark),
C.ushort(t.Tmtu),
C.ushort(t.VxlanPort),
C.ushort(t.PSNatStart),
C.ushort(t.PSNatLen),
(*C.char)(unsafe.Pointer(&t.HostTunnelIPv4[0])),
(*C.char)(unsafe.Pointer(&t.HostTunnelIPv6[0])),
C.uint(t.Flags),
C.ushort(t.WgPort),
C.ushort(t.Wg6Port),
C.ushort(t.Profiling),
C.uint(t.NatIn),
C.uint(t.NatOut),
C.uint(t.LogFilterJmp),
&cJumps[0], // it is safe because we hold the reference here until we return.
&cJumpsV6[0],
)

return err
}

func CTLBSetGlobals(m *Map, udpNotSeen time.Duration, excludeUDP bool) error {
udpNotSeen /= time.Second // Convert to seconds
_, err := C.bpf_ctlb_set_globals(m.bpfMap, C.uint(udpNotSeen), C.bool(excludeUDP))
func (c *CTCleanupGlobalData) Set(m *Map) error {
_, err := C.bpf_ct_cleanup_set_globals(
m.bpfMap,
C.uint64_t(c.CreationGracePeriod.Nanoseconds()),

C.uint64_t(c.TCPPreEstablished.Nanoseconds()),
C.uint64_t(c.TCPEstablished.Nanoseconds()),
C.uint64_t(c.TCPFinsSeen.Nanoseconds()),
C.uint64_t(c.TCPResetSeen.Nanoseconds()),

C.uint64_t(c.UDPLastSeen.Nanoseconds()),
C.uint64_t(c.GenericIPLastSeen.Nanoseconds()),
C.uint64_t(c.ICMPLastSeen.Nanoseconds()),
)
return err
}

func XDPSetGlobals(
m *Map,
globalData *XDPGlobalData,
) error {
func (c *CTLBGlobalData) Set(m *Map) error {
udpNotSeen := c.UDPNotSeen / time.Second // Convert to seconds
_, err := C.bpf_ctlb_set_globals(m.bpfMap, C.uint(udpNotSeen), C.bool(c.ExcludeUDP))

return err
}

cName := C.CString(globalData.IfaceName)
func (x *XDPGlobalData) Set(m *Map) error {
cName := C.CString(x.IfaceName)
defer C.free(unsafe.Pointer(cName))

cJumps := make([]C.uint, len(globalData.Jumps))
cJumpsV6 := make([]C.uint, len(globalData.Jumps))
cJumps := make([]C.uint, len(x.Jumps))
cJumpsV6 := make([]C.uint, len(x.Jumps))

for i, v := range globalData.Jumps {
for i, v := range x.Jumps {
cJumps[i] = C.uint(v)
}

for i, v := range globalData.JumpsV6 {
for i, v := range x.JumpsV6 {
cJumpsV6[i] = C.uint(v)
}
_, err := C.bpf_xdp_set_globals(m.bpfMap,
Expand Down
24 changes: 24 additions & 0 deletions felix/bpf/libbpf/libbpf_common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@

package libbpf

import (
"time"
)

type GlobalData interface {
Set(m *Map) error
}

type TcGlobalData struct {
IfaceName string
HostIPv4 [16]byte
Expand Down Expand Up @@ -44,3 +52,19 @@ type XDPGlobalData struct {
Jumps [16]uint32
JumpsV6 [16]uint32
}

type CTCleanupGlobalData struct {
CreationGracePeriod time.Duration
TCPPreEstablished time.Duration
TCPEstablished time.Duration
TCPFinsSeen time.Duration
TCPResetSeen time.Duration
UDPLastSeen time.Duration
GenericIPLastSeen time.Duration
ICMPLastSeen time.Duration
}

type CTLBGlobalData struct {
UDPNotSeen time.Duration
ExcludeUDP bool
}
Loading

0 comments on commit 83e65f6

Please sign in to comment.