Skip to content

Commit 43f8847

Browse files
author
KJ Tsanaktsidis
committedNov 19, 2021
Find server PIDs based on socket inode number
1 parent 2eaa236 commit 43f8847

6 files changed

+190
-4
lines changed
 

‎Dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -mod=vendor -ldflags "-X main.version=$
1111
raingutter/raingutter.go \
1212
raingutter/socket_stats.go \
1313
raingutter/prometheus.go \
14-
raingutter/netlink_socket_stats.go
14+
raingutter/netlink_socket_stats.go \
15+
raingutter/process.go
1516

1617
FROM scratch
1718

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ The following environment variables can be used to configure Raingutter:
4141
* `RG_USE_SOCKET_STATS`: Deprecated - use `RG_SOCKET_STATS_MODE` instead. If set to `true`, will behave like `RG_SOCKET_STATS_MODE == "proc_net"`, and if set to `false` will behave like `RG_SOCKET_STATS_MODE == "raindrops"`.
4242
* `RG_FREQUENCY`: Polling frequency in milliseconds (default: `500`)
4343
* `RG_SERVER_PORT`: Where the web server listens to
44+
* `RG_MEMORY_STATS_ENABLED`: If enabled, attempt to collect memory usage statistics for processers listening on `RG_SERVER_PORT`. This is most useful for preforking webservers like unicorn, where it will measure how much memory is copy-on-write shared between processes. If using this feature, you should NOT use `RG_SOCKET_STATS_MODE=netlink` - Raingutter relies on lining up the listener socket inode numbers with `/proc/$pid/fd/` to find out which processes are listening on a socket. The inode is stored in the kernel as a 64-bit integer, however the INET_DIAG netlink API only exposes it as a 32-bit integer, doing silent wrap around! This means that if you use `RG_SOCKET_STATS_MODE=netlink`, `RG_MEMORY_STATS_ENABLED` might simply fail to generate any metrics at all if your system has had a lot of sockets.
4445
* `RG_PROC_DIRECTORY`: Path to `/proc` directory to use. Useful when running in a container to point to a path where the host's `/proc` directory is mounted.
4546

4647
##### Pre-fork web servers (Unicorn)

‎raingutter/netlink_socket_stats_test.go

+14-3
Original file line numberDiff line numberDiff line change
@@ -11,31 +11,42 @@ import (
1111
"time"
1212
)
1313

14-
func testNetlinkSocketStatsImpl(t *testing.T, bindAddr string) {
15-
// Set up a listener socket
14+
func listenAndGetInodeNumber(t *testing.T, bindAddr string) (*net.TCPListener, uint64) {
1615
l, err := net.Listen("tcp", bindAddr)
1716
if err != nil {
1817
t.Fatalf("failed to listen: %s", err)
1918
}
20-
port := uint16(l.Addr().(*net.TCPAddr).Port)
2119
listenerFd, err := l.(*net.TCPListener).File()
2220
if err != nil {
21+
l.Close()
2322
t.Fatalf("failed to get fd for listener: %s", err)
2423
}
2524
listenerInodeStr, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", listenerFd.Fd()))
2625
if err != nil {
26+
l.Close()
2727
t.Fatalf("failed to readlink fd for listener: %s", err)
2828
}
2929
socketIndoeRegexp := regexp.MustCompile(`socket:\[([0-9]+)\]`)
3030
matches := socketIndoeRegexp.FindStringSubmatch(listenerInodeStr)
3131
if len(matches) < 2 {
32+
l.Close()
3233
t.Fatalf("could not parse socket inode %s", listenerInodeStr)
3334
}
3435
listenerInode, err := strconv.Atoi(matches[1])
3536
if err != nil {
37+
l.Close()
3638
t.Fatalf("could not convert socket inode %s to int: %s", matches[1], err)
3739
}
3840

41+
return l.(*net.TCPListener), uint64(listenerInode)
42+
}
43+
44+
func testNetlinkSocketStatsImpl(t *testing.T, bindAddr string) {
45+
// Set up a listener socket
46+
l, listenerInode := listenAndGetInodeNumber(t, bindAddr)
47+
defer l.Close()
48+
port := uint16(l.Addr().(*net.TCPAddr).Port)
49+
3950
rnlc, err := NewRaingutterNetlinkConnection()
4051
if err != nil {
4152
t.Fatalf("failed to create netlink connection: %s", err)

‎raingutter/process.go

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path"
7+
"strconv"
8+
"unsafe"
9+
10+
"golang.org/x/sys/unix"
11+
)
12+
13+
type ServerProcess struct {
14+
Pid int
15+
}
16+
17+
// linuxDirent64 is linux_dirent64 from linux/dirent.h
18+
type linuxDirent64 struct {
19+
DIno uint64 // u64 d_ino
20+
DOff int64 // s64 d_off
21+
DReclen uint16 // unsigned short d_reclen
22+
DType uint8 // unsigned char d_type
23+
DName byte // The first byte of flexible array member char d_name[]
24+
}
25+
26+
func FindProcessesListeningToSocket(procDir string, socketInode uint64) ([]ServerProcess, error) {
27+
// Check what network namespace _we_ are in.
28+
selfNetNs, err := os.Readlink(path.Join(procDir, "self/ns/net"))
29+
if err != nil {
30+
return nil, fmt.Errorf("error reading %s/self/ns/net: %w", procDir, err)
31+
}
32+
33+
// List out every process in /proc
34+
procEntries, err := os.ReadDir(procDir)
35+
if err != nil {
36+
return nil, fmt.Errorf("error reading dir %s: %w", procDir, err)
37+
}
38+
39+
var result []ServerProcess
40+
41+
for _, entry := range procEntries {
42+
// use an IIFE so that we can defer closing the directory FD.
43+
func(){
44+
if !entry.IsDir() {
45+
return
46+
}
47+
pid, err := strconv.Atoi(entry.Name())
48+
if err != nil {
49+
// Not a proc/$PID directory
50+
return
51+
}
52+
dirFD, err := os.OpenFile(path.Join(procDir, entry.Name()), os.O_RDONLY | unix.O_DIRECTORY, 0)
53+
if err != nil {
54+
// failed to open the directory - process may simply have died before we could open it.
55+
return
56+
}
57+
defer dirFD.Close()
58+
59+
// Is this process in the same namespace as us?
60+
// format of this link is "net:[uint64 in base10]" so this buffer is always
61+
// going to be large enough
62+
linkBuffer := make([]byte, 64)
63+
n, err := unix.Readlinkat(int(dirFD.Fd()), "ns/net", linkBuffer)
64+
if err != nil {
65+
// Could mean a couple of things - this process might have exited, or we might not
66+
// be in the same (or parent) net namespace as the pid. We want to filter for processes
67+
// in the same network namespace anyway, so ignore this regardless.
68+
return
69+
}
70+
pidNetNs := string(linkBuffer[:n])
71+
if selfNetNs != pidNetNs {
72+
// this process is in a different network namespace. Ignore.
73+
return
74+
}
75+
76+
77+
// See if it has an open file with the given socketInode.
78+
fdDirFD, err := unix.Openat(int(dirFD.Fd()), "fd", os.O_RDONLY | unix.O_DIRECTORY, 0)
79+
if err != nil {
80+
// could have exited
81+
return
82+
}
83+
defer unix.Close(fdDirFD)
84+
85+
pidHasListenerSocketOpened := false
86+
// Gross-time. Golang has no binding for "read entries from a directory FD". This is important,
87+
// because pids can be recycled, so doing something like os.Readdir("/proc/$pid/fd") is racey.
88+
// Linux has the getdents64 syscall for this purpose; use the raw syscall API.
89+
dentBuf := make([]byte, 4096)
90+
for {
91+
nBytes, err := unix.Getdents(fdDirFD, dentBuf)
92+
if err != nil {
93+
return
94+
}
95+
if nBytes == 0 {
96+
break
97+
}
98+
offset := 0
99+
for offset < nBytes {
100+
dentStruct := (*linuxDirent64)(unsafe.Pointer(&dentBuf[offset]))
101+
102+
fdFileNameStartOffset := offset + int(unsafe.Offsetof(dentStruct.DName))
103+
fdFileNameEndOffset := fdFileNameStartOffset
104+
for dentBuf[fdFileNameEndOffset] != 0 {
105+
fdFileNameEndOffset++
106+
}
107+
fdFileName := string(dentBuf[fdFileNameStartOffset:fdFileNameEndOffset])
108+
109+
linkBytes, err := unix.Readlinkat(fdDirFD, fdFileName, linkBuffer)
110+
if err == nil && string(linkBuffer[:linkBytes]) == fmt.Sprintf("socket:[%d]", socketInode) {
111+
// This process has the listener socket opened.
112+
pidHasListenerSocketOpened = true
113+
}
114+
115+
offset += int(dentStruct.DReclen)
116+
}
117+
}
118+
119+
if pidHasListenerSocketOpened {
120+
result = append(result, ServerProcess{
121+
Pid: pid,
122+
})
123+
}
124+
}()
125+
}
126+
return result, nil
127+
}

‎raingutter/process_test.go

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"testing"
6+
)
7+
8+
func TestFindProcessesListeningToSocket(t *testing.T) {
9+
// Open a listening socket.
10+
l, listenerInode := listenAndGetInodeNumber(t, "127.0.0.1:0")
11+
defer l.Close()
12+
13+
// Find processes using that socket
14+
procs, err := FindProcessesListeningToSocket("/proc", listenerInode)
15+
if err != nil {
16+
t.Fatalf("error FindProcessesListeningToSocket: %s", err)
17+
}
18+
19+
// This process should be the only process using that socket.
20+
if len(procs) != 1 {
21+
t.Fail()
22+
}
23+
if procs[0].Pid != os.Getpid() {
24+
t.Fail()
25+
}
26+
}

‎raingutter/raingutter.go

+20
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,13 @@ func main() {
329329
log.Info("RG_FREQUENCY: ", frequency)
330330
freqInt, _ := strconv.Atoi(frequency)
331331

332+
333+
memoryStatsEnabledStr := os.Getenv("RG_MEMORY_STATS_ENABLED")
334+
memoryStatsEnabled := false
335+
if strings.ToLower(memoryStatsEnabledStr) == "true" {
336+
memoryStatsEnabled = true
337+
}
338+
332339
if podName == "" {
333340
log.Warn("POD_NAME is missing")
334341
}
@@ -450,6 +457,19 @@ func main() {
450457
}
451458
}
452459

460+
if memoryStatsEnabled {
461+
serverProcs, err := FindProcessesListeningToSocket(procDir, r.ListenerSocketInode)
462+
if err != nil {
463+
log.Error(err)
464+
} else {
465+
var pids []int
466+
for _, s := range serverProcs {
467+
pids = append(pids, s.Pid)
468+
}
469+
log.Infof("Found unicorn pids: %+v", pids)
470+
}
471+
}
472+
453473
if didScan {
454474
if statsdEnabled == "true" {
455475
r.sendStats(statsdClient, &tc, useThreads)

0 commit comments

Comments
 (0)
Please sign in to comment.