Skip to content

Commit a49ec4d

Browse files
konradasb-hostingerkonradasb-hostinger
authored
feat(server): Add configurable network rate limiting for MicroVMs (#341)
1 parent 6ae624e commit a49ec4d

6 files changed

Lines changed: 231 additions & 16 deletions

File tree

‎docs/reference/configuration.md‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,67 @@ pools:
147147
#
148148
vcpu_count: 2
149149
#
150+
# Firecracker network interface configuration.
151+
#
152+
# Default: {} (no rate limiting)
153+
#
154+
network_interface:
155+
#
156+
# Rate limiter for incoming (ingress) traffic. Maps to the Firecracker
157+
# network interface `rx_rate_limiter`. Both the `bandwidth` and `ops`
158+
# token buckets are optional, an omitted bucket means unlimited.
159+
#
160+
# The resulting rate is `size` / `refill_time`.
161+
#
162+
# Default: {} (unlimited)
163+
#
164+
in_rate_limiter:
165+
#
166+
# Token bucket with bytes as tokens.
167+
#
168+
# Default: {} (unlimited)
169+
#
170+
bandwidth:
171+
#
172+
# The total number of tokens (bytes) the bucket can hold.
173+
#
174+
# Required: true
175+
#
176+
size: 131072000
177+
#
178+
# The amount of milliseconds it takes for the bucket to refill.
179+
# 131072000 bytes per 1000 ms is ~125 MiB/s.
180+
#
181+
# Required: true
182+
#
183+
refill_time: 1000
184+
#
185+
# The initial burst size (bytes). Consumed before the refill process
186+
# starts happening.
187+
#
188+
# Default: 0
189+
#
190+
one_time_burst: 262144000
191+
#
192+
# Token bucket with operations (packets) as tokens.
193+
#
194+
# Default: {} (unlimited)
195+
#
196+
ops:
197+
size: 10000
198+
refill_time: 1000
199+
#
200+
# Rate limiter for outgoing (egress) traffic. Maps to the Firecracker
201+
# network interface `tx_rate_limiter`. Same structure as
202+
# `in_rate_limiter`.
203+
#
204+
# Default: {} (unlimited)
205+
#
206+
out_rate_limiter:
207+
bandwidth:
208+
size: 26214400
209+
refill_time: 1000
210+
#
150211
# Metadata to pass to the Firecracker VM via MMDS.
151212
#
152213
# Default: {}

‎server/config.go‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"fmt"
55
"os"
66

7+
"github.com/firecracker-microvm/firecracker-go-sdk"
8+
"github.com/firecracker-microvm/firecracker-go-sdk/client/models"
79
"github.com/go-playground/validator/v10"
810
"gopkg.in/yaml.v3"
911
)
@@ -16,7 +18,7 @@ type Config struct {
1618
BasicAuthEnabled bool `yaml:"basic_auth_enabled" validate:""`
1719
BasicAuthUsers map[string]string `yaml:"basic_auth_users" validate:"required_if=basic_auth_enabled true"`
1820
GitHub *GitHubConfig `yaml:"github" validate:"required"`
19-
Pools []*PoolConfig `yaml:"pools" validate:"required,min=1"`
21+
Pools []*PoolConfig `yaml:"pools" validate:"required,min=1,dive,required"`
2022
LogLevel string `yaml:"log_level" validate:"required,oneof=debug info warn error fatal panic trace"`
2123

2224
path string
@@ -47,18 +49,67 @@ type RunnerConfig struct {
4749
}
4850

4951
type FirecrackerConfig struct {
50-
BinaryPath string `yaml:"binary_path" `
51-
KernelImagePath string `yaml:"kernel_image_path"`
52-
KernelArgs string `yaml:"kernel_args"`
53-
MachineConfig FirecrackerMachineConfig `yaml:"machine_config"`
54-
Metadata map[string]interface{} `yaml:"metadata"`
52+
BinaryPath string `yaml:"binary_path" `
53+
KernelImagePath string `yaml:"kernel_image_path"`
54+
KernelArgs string `yaml:"kernel_args"`
55+
MachineConfig FirecrackerMachineConfig `yaml:"machine_config"`
56+
NetworkInterface *FirecrackerNetworkInterfaceConfig `yaml:"network_interface"`
57+
Metadata map[string]interface{} `yaml:"metadata"`
5558
}
5659

5760
type FirecrackerMachineConfig struct {
5861
VcpuCount int64 `yaml:"vcpu_count"`
5962
MemSizeMib int64 `yaml:"mem_size_mib"`
6063
}
6164

65+
// FirecrackerNetworkInterfaceConfig configures the MicroVM's network interface.
66+
// Rate limiters are optional, a nil limiter leaves that direction unlimited.
67+
type FirecrackerNetworkInterfaceConfig struct {
68+
InRateLimiter *FirecrackerRateLimiterConfig `yaml:"in_rate_limiter"`
69+
OutRateLimiter *FirecrackerRateLimiterConfig `yaml:"out_rate_limiter"`
70+
}
71+
72+
// FirecrackerRateLimiterConfig defines an IO rate limiter with independent
73+
// bytes/s and ops/s limits. A nil token bucket leaves that limit unlimited.
74+
type FirecrackerRateLimiterConfig struct {
75+
Bandwidth *FirecrackerTokenBucketConfig `yaml:"bandwidth"`
76+
Ops *FirecrackerTokenBucketConfig `yaml:"ops"`
77+
}
78+
79+
// FirecrackerTokenBucketConfig defines a token bucket with a maximum capacity
80+
// (Size), an optional initial burst size (OneTimeBurst) and the interval in
81+
// milliseconds it takes to refill the bucket (RefillTime). The resulting rate
82+
// is Size / RefillTime.
83+
type FirecrackerTokenBucketConfig struct {
84+
Size int64 `yaml:"size" validate:"required,gt=0"`
85+
OneTimeBurst *int64 `yaml:"one_time_burst" validate:"omitempty,gte=0"`
86+
RefillTime int64 `yaml:"refill_time" validate:"required,gt=0"`
87+
}
88+
89+
// toSDK converts the rate limiter configuration into its Firecracker SDK
90+
// representation. Returns nil if the rate limiter isn't configured.
91+
func (c *FirecrackerRateLimiterConfig) toSDK() *models.RateLimiter {
92+
if c == nil {
93+
return nil
94+
}
95+
96+
return &models.RateLimiter{Bandwidth: c.Bandwidth.toSDK(), Ops: c.Ops.toSDK()}
97+
}
98+
99+
// toSDK converts the token bucket configuration into its Firecracker SDK
100+
// representation. Returns nil if the token bucket isn't configured.
101+
func (c *FirecrackerTokenBucketConfig) toSDK() *models.TokenBucket {
102+
if c == nil {
103+
return nil
104+
}
105+
106+
return &models.TokenBucket{
107+
Size: firecracker.Int64(c.Size),
108+
RefillTime: firecracker.Int64(c.RefillTime),
109+
OneTimeBurst: c.OneTimeBurst,
110+
}
111+
}
112+
62113
// DefaultConfig creates a new Config with default values.
63114
func DefaultConfig() *Config {
64115
c := &Config{

‎server/config_test.go‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package server
33
import (
44
"testing"
55

6+
"github.com/firecracker-microvm/firecracker-go-sdk"
7+
"github.com/firecracker-microvm/firecracker-go-sdk/client/models"
68
"github.com/stretchr/testify/assert"
79
)
810

@@ -14,3 +16,52 @@ func TestNewConfig(t *testing.T) {
1416

1517
assert.Equal(t, "testdata/config1.yaml", config.path)
1618
}
19+
20+
func TestNewConfigNetworkInterface(t *testing.T) {
21+
config, err := NewConfig("testdata/config1.yaml")
22+
if err != nil {
23+
t.Fatalf("unexpected error: %v", err)
24+
}
25+
26+
networkInterface := config.Pools[0].Firecracker.NetworkInterface
27+
if networkInterface == nil {
28+
t.Fatal("expected network interface configuration to be set")
29+
}
30+
31+
assert.Equal(t, &FirecrackerTokenBucketConfig{
32+
Size: 131072000, RefillTime: 1000, OneTimeBurst: firecracker.Int64(262144000),
33+
}, networkInterface.InRateLimiter.Bandwidth)
34+
assert.Equal(t, &FirecrackerTokenBucketConfig{
35+
Size: 10000, RefillTime: 1000,
36+
}, networkInterface.InRateLimiter.Ops)
37+
assert.Equal(t, &FirecrackerTokenBucketConfig{
38+
Size: 26214400, RefillTime: 1000,
39+
}, networkInterface.OutRateLimiter.Bandwidth)
40+
assert.Nil(t, networkInterface.OutRateLimiter.Ops)
41+
42+
// A pool without a network_interface block leaves the interface unlimited.
43+
assert.Nil(t, config.Pools[1].Firecracker.NetworkInterface)
44+
}
45+
46+
func TestNewConfigNetworkInterfaceInvalid(t *testing.T) {
47+
// A token bucket without a size is rejected.
48+
_, err := NewConfig("testdata/config2.yaml")
49+
assert.ErrorContains(t, err, "Config.Pools[0].Firecracker.NetworkInterface.InRateLimiter.Bandwidth.Size")
50+
}
51+
52+
func TestFirecrackerRateLimiterConfigToSDK(t *testing.T) {
53+
var nilRateLimiter *FirecrackerRateLimiterConfig
54+
assert.Nil(t, nilRateLimiter.toSDK())
55+
56+
rateLimiter := &FirecrackerRateLimiterConfig{
57+
Bandwidth: &FirecrackerTokenBucketConfig{Size: 131072000, RefillTime: 1000, OneTimeBurst: firecracker.Int64(262144000)},
58+
}
59+
60+
assert.Equal(t, &models.RateLimiter{
61+
Bandwidth: &models.TokenBucket{
62+
Size: firecracker.Int64(131072000),
63+
RefillTime: firecracker.Int64(1000),
64+
OneTimeBurst: firecracker.Int64(262144000),
65+
},
66+
}, rateLimiter.toSDK())
67+
}

‎server/pool.go‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,16 @@ func (p *Pool) createMachine(ctx context.Context) error {
461461
vsockPath := filepath.Join(p.GetDir(), fmt.Sprintf("%s.vsock", runnerName))
462462
vsockCID := p.nextCID.Add(1)
463463

464+
networkInterface := firecracker.NetworkInterface{
465+
AllowMMDS: true,
466+
CNIConfiguration: &firecracker.CNIConfiguration{NetworkName: "fireactions", IfName: "eth0", ConfDir: "/etc/cni/net.d", BinPath: []string{"/opt/cni/bin"}},
467+
}
468+
469+
if networkInterfaceConfig := p.config.Firecracker.NetworkInterface; networkInterfaceConfig != nil {
470+
networkInterface.InRateLimiter = networkInterfaceConfig.InRateLimiter.toSDK()
471+
networkInterface.OutRateLimiter = networkInterfaceConfig.OutRateLimiter.toSDK()
472+
}
473+
464474
fcMachine, err := firecracker.NewMachine(ctx, firecracker.Config{
465475
VMID: runnerName,
466476
SocketPath: filepath.Join(p.GetDir(), fmt.Sprintf("%s.sock", runnerName)),
@@ -476,16 +486,13 @@ func (p *Pool) createMachine(ctx context.Context) error {
476486
IsRootDevice: firecracker.Bool(true),
477487
IsReadOnly: firecracker.Bool(false),
478488
}},
479-
NetworkInterfaces: []firecracker.NetworkInterface{{
480-
AllowMMDS: true,
481-
CNIConfiguration: &firecracker.CNIConfiguration{NetworkName: "fireactions", IfName: "eth0", ConfDir: "/etc/cni/net.d", BinPath: []string{"/opt/cni/bin"}},
482-
}},
483-
VsockDevices: []firecracker.VsockDevice{{Path: vsockPath, CID: vsockCID}},
484-
MmdsAddress: net.IPv4(169, 254, 169, 254),
485-
MmdsVersion: firecracker.MMDSv2,
486-
ForwardSignals: []os.Signal{},
487-
LogPath: filepath.Join(p.GetDir(), fmt.Sprintf("%s.firecracker.log", runnerName)),
488-
LogLevel: "Debug",
489+
NetworkInterfaces: []firecracker.NetworkInterface{networkInterface},
490+
VsockDevices: []firecracker.VsockDevice{{Path: vsockPath, CID: vsockCID}},
491+
MmdsAddress: net.IPv4(169, 254, 169, 254),
492+
MmdsVersion: firecracker.MMDSv2,
493+
ForwardSignals: []os.Signal{},
494+
LogPath: filepath.Join(p.GetDir(), fmt.Sprintf("%s.firecracker.log", runnerName)),
495+
LogLevel: "Debug",
489496
}, firecracker.WithProcessRunner(machineCmd), firecracker.WithLogger(logrus.NewEntry(logger)))
490497
if err != nil {
491498
return fmt.Errorf("firecracker: creating machine: %w", err)

‎server/testdata/config1.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,19 @@ pools:
3737
machine_config:
3838
mem_size_mib: 2048
3939
vcpu_count: 2
40+
network_interface:
41+
in_rate_limiter:
42+
bandwidth:
43+
size: 131072000
44+
refill_time: 1000
45+
one_time_burst: 262144000
46+
ops:
47+
size: 10000
48+
refill_time: 1000
49+
out_rate_limiter:
50+
bandwidth:
51+
size: 26214400
52+
refill_time: 1000
4053
metadata:
4154
example1: value1
4255
- name: fireactions-2vcpu-4gb

‎server/testdata/config2.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
---
2+
bind_address: 0.0.0.0:8080
3+
4+
github:
5+
app_private_key: |
6+
-----BEGIN RSA PRIVATE KEY-----
7+
app_id: 12345
8+
9+
pools:
10+
- name: fireactions-2vcpu-2gb
11+
replicas: 1
12+
runner:
13+
name: fireactions-2vcpu-2gb
14+
image: ghcr.io/hostinger/fireactions/runner:ubuntu-20.04-x64-2.310.2
15+
image_pull_policy: IfNotPresent
16+
group_id: 1
17+
organization: hostinger
18+
labels:
19+
- self-hosted
20+
firecracker:
21+
binary_path: firecracker
22+
kernel_image_path: /var/lib/fireactions/vmlinux
23+
kernel_args: "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules rw"
24+
machine_config:
25+
mem_size_mib: 2048
26+
vcpu_count: 2
27+
network_interface:
28+
in_rate_limiter:
29+
bandwidth:
30+
refill_time: 1000
31+
32+
log_level: debug

0 commit comments

Comments
 (0)