diff --git a/CubeOps/config.example.yaml b/CubeOps/config.example.yaml index 7a10bf1c3..ed03c27fd 100644 --- a/CubeOps/config.example.yaml +++ b/CubeOps/config.example.yaml @@ -11,6 +11,22 @@ log_file_num: 10 # number of rotated log files to retain log_file_size: 100 # max size in MB per log file before rotation jwt_secret: "" # leave empty to auto-generate on first start +# Reverse proxies allowed to assert the client IP via X-Forwarded-For. +# Empty (the default) trusts none: the login rate limiter keys on the real TCP +# peer, which is correct for direct exposure. Behind nginx/an LB, list the +# proxy here as an IP or CIDR, otherwise every client shares one bucket. +# Keep this as narrow as possible -- any peer whose source address matches is +# trusted to assert any client IP, so a broad range such as 10.0.0.0/8 hands +# that power to every workload on the network. 0.0.0.0/0 and ::/0 are rejected, +# and ranges wider than /16 (v4) or /64 (v6) are accepted but warned about at +# startup. An empty list is also warned about, since it means every client +# behind a proxy shares one login bucket. +# Note: gin expands a bare loopback entry such as 127.0.0.1 to the whole +# 127.0.0.0/8, and in containerised deployments the proxy usually connects from +# a bridge address rather than loopback, so 127.0.0.1 is often not what you want. +# Env override: CUBE_OPS_TRUSTED_PROXIES (comma-separated). +trusted_proxies: [] + # --- Database (MySQL) --- # IMPORTANT: Replace the placeholder values below with your real credentials # before deploying. The defaults shown here are EXAMPLES ONLY and will not diff --git a/CubeOps/e2e/doc.go b/CubeOps/e2e/doc.go new file mode 100644 index 000000000..a2482f79e --- /dev/null +++ b/CubeOps/e2e/doc.go @@ -0,0 +1,12 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package e2e holds end-to-end tests that drive the real cubeops binary against +// a real database over HTTP. +// +// Every test file in this package carries the e2e build tag, so the default +// unit gate (go test ./...) compiles this file and reports "no test files" +// instead of failing on a package with no buildable sources. Run the suite with: +// +// go test -tags e2e ./e2e/... +package e2e diff --git a/CubeOps/e2e/harness_test.go b/CubeOps/e2e/harness_test.go new file mode 100644 index 000000000..e3603868c --- /dev/null +++ b/CubeOps/e2e/harness_test.go @@ -0,0 +1,283 @@ +//go:build e2e + +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package e2e drives the real cubeops binary against a real MySQL instance over +// HTTP. It is excluded from the default unit gate by the e2e build tag; run it +// with: +// +// cd CubeOps && go test -tags e2e ./e2e/... +package e2e + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +const ( + requireDockerEnv = "CUBEOPS_REQUIRE_DOCKER_TESTS" + mysqlTag = "8.0" + dbName = "cubeops_e2e" +) + +type env struct { + t *testing.T + binary string + dsn string + mysqlURL struct{ host, port string } + logDir string + teardown func() +} + +func requireDocker() bool { + v := os.Getenv(requireDockerEnv) + if v == "1" || strings.EqualFold(v, "true") { + return true + } + ci := os.Getenv("CI") + return ci == "true" || ci == "1" +} + +func abortOrSkip(t *testing.T, format string, args ...any) { + t.Helper() + msg := fmt.Sprintf(format, args...) + if requireDocker() { + t.Fatalf("%s (set %s or fix Docker — CI forbids skip)", msg, requireDockerEnv) + } + t.Skipf("%s", msg) +} + +// newEnv builds the cubeops binary and starts a throwaway MySQL container. +func newEnv(t *testing.T) *env { + t.Helper() + + pool, err := dockertest.NewPool("") + if err != nil { + abortOrSkip(t, "dockertest not available (%v)", err) + } + if err := pool.Client.Ping(); err != nil { + abortOrSkip(t, "docker daemon not reachable (%v)", err) + } + + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "mysql", + Tag: mysqlTag, + Env: []string{ + "MYSQL_ROOT_PASSWORD=root", + "MYSQL_DATABASE=" + dbName, + }, + }, func(hc *docker.HostConfig) { + hc.AutoRemove = true + hc.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + abortOrSkip(t, "could not start mysql container (%v)", err) + } + + port := resource.GetPort("3306/tcp") + dsn := fmt.Sprintf("root:root@tcp(127.0.0.1:%s)/%s?charset=utf8&parseTime=true", port, dbName) + + // MySQL's entrypoint runs a temporary server during initialisation and then + // restarts, so the first successful connection can still be to the throwaway + // instance. Require a working query, then a short settle, before proceeding. + pool.MaxWait = 3 * time.Minute + if err := pool.Retry(func() error { + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer db.Close() + var one int + return db.QueryRow("SELECT 1").Scan(&one) + }); err != nil { + _ = pool.Purge(resource) + t.Fatalf("mysql never became reachable: %v", err) + } + time.Sleep(2 * time.Second) + + binary := filepath.Join(t.TempDir(), "cubeops") + build := exec.Command("go", "build", "-o", binary, "./cmd/cubeops") + build.Dir = ".." + if out, err := build.CombinedOutput(); err != nil { + _ = pool.Purge(resource) + t.Fatalf("building cubeops failed: %v\n%s", err, out) + } + + e := &env{t: t, binary: binary, dsn: dsn, logDir: t.TempDir()} + e.mysqlURL.host, e.mysqlURL.port = "127.0.0.1", port + e.teardown = func() { _ = pool.Purge(resource) } + return e +} + +func freePort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserve port: %v", err) + } + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +type instance struct { + baseURL string + cmd *exec.Cmd + logPath string +} + +// start launches cubeops and waits for /health. extraEnv entries are KEY=VALUE. +func (e *env) start(t *testing.T, extraEnv ...string) *instance { + t.Helper() + port := freePort(t) + logPath := filepath.Join(e.logDir, fmt.Sprintf("cubeops-%d.out", port)) + out, err := os.Create(logPath) + if err != nil { + t.Fatalf("create log: %v", err) + } + + cmd := exec.Command(e.binary) + cmd.Env = append(os.Environ(), + fmt.Sprintf("CUBE_OPS_BIND=127.0.0.1:%d", port), + "CUBE_SANDBOX_MYSQL_HOST="+e.mysqlURL.host, + "CUBE_SANDBOX_MYSQL_PORT="+e.mysqlURL.port, + "CUBE_SANDBOX_MYSQL_USER=root", + "CUBE_SANDBOX_MYSQL_PASSWORD=root", + "CUBE_SANDBOX_MYSQL_DB="+dbName, + "CUBE_OPS_LOG_DIR="+filepath.Join(e.logDir, fmt.Sprintf("log-%d", port)), + ) + cmd.Env = append(cmd.Env, extraEnv...) + cmd.Stdout, cmd.Stderr = out, out + if err := cmd.Start(); err != nil { + t.Fatalf("start cubeops: %v", err) + } + + inst := &instance{baseURL: fmt.Sprintf("http://127.0.0.1:%d", port), cmd: cmd, logPath: logPath} + deadline := time.Now().Add(4 * time.Minute) + for time.Now().Before(deadline) { + if resp, err := http.Get(inst.baseURL + "/health"); err == nil { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return inst + } + } + if cmd.ProcessState != nil && cmd.ProcessState.Exited() { + break + } + time.Sleep(500 * time.Millisecond) + } + body, _ := os.ReadFile(logPath) + inst.stop() + t.Fatalf("cubeops never became healthy on %s\n--- output ---\n%s", inst.baseURL, tailLines(string(body), 25)) + return nil +} + +// startExpectingExit launches cubeops and waits for it to terminate, returning +// its combined output. It fails the test if the process stays up. +func (e *env) startExpectingExit(t *testing.T, extraEnv ...string) string { + t.Helper() + port := freePort(t) + cmd := exec.Command(e.binary) + cmd.Env = append(os.Environ(), + fmt.Sprintf("CUBE_OPS_BIND=127.0.0.1:%d", port), + "CUBE_SANDBOX_MYSQL_HOST="+e.mysqlURL.host, + "CUBE_SANDBOX_MYSQL_PORT="+e.mysqlURL.port, + "CUBE_SANDBOX_MYSQL_USER=root", + "CUBE_SANDBOX_MYSQL_PASSWORD=root", + "CUBE_SANDBOX_MYSQL_DB="+dbName, + "CUBE_OPS_LOG_DIR="+filepath.Join(e.logDir, fmt.Sprintf("exitlog-%d", port)), + ) + cmd.Env = append(cmd.Env, extraEnv...) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("cubeops kept running; expected it to abort\n%s", tailLines(string(out), 20)) + } + return string(out) +} + +func (i *instance) stop() { + if i == nil || i.cmd == nil || i.cmd.Process == nil { + return + } + _ = i.cmd.Process.Kill() + _, _ = i.cmd.Process.Wait() +} + +func tailLines(s string, n int) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} + +func (e *env) db(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("mysql", e.dsn) + if err != nil { + t.Fatalf("open db: %v", err) + } + return db +} + +// do issues a request and returns the status and body. +func do(t *testing.T, method, url, bearer, body string) (int, string) { + t.Helper() + var rdr io.Reader + if body != "" { + rdr = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(context.Background(), method, url, rdr) + if err != nil { + t.Fatalf("build request: %v", err) + } + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(raw) +} + +// login performs a real login and returns the access token. +func login(t *testing.T, baseURL, user, pass string) string { + t.Helper() + code, body := do(t, http.MethodPost, baseURL+"/api/v1/auth/login", "", + fmt.Sprintf(`{"username":%q,"password":%q}`, user, pass)) + if code != http.StatusOK { + t.Fatalf("login returned %d: %s", code, body) + } + var parsed map[string]any + if err := json.Unmarshal([]byte(body), &parsed); err != nil { + t.Fatalf("login response is not JSON: %s", body) + } + for _, key := range []string{"accessToken", "access_token"} { + if v, ok := parsed[key].(string); ok && v != "" { + return v + } + } + t.Fatalf("no access token in login response: %s", body) + return "" +} diff --git a/CubeOps/e2e/login_ratelimit_xff_test.go b/CubeOps/e2e/login_ratelimit_xff_test.go new file mode 100644 index 000000000..44195f1b2 --- /dev/null +++ b/CubeOps/e2e/login_ratelimit_xff_test.go @@ -0,0 +1,125 @@ +//go:build e2e + +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" +) + +// badLogin sends one failing login attempt, optionally asserting a client IP via +// X-Forwarded-For, and returns the HTTP status. +func badLogin(t *testing.T, baseURL, forwardedFor string) int { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + baseURL+"/api/v1/auth/login", + strings.NewReader(`{"username":"admin","password":"definitely-not-the-password"}`)) + if err != nil { + t.Fatalf("build request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + if forwardedFor != "" { + req.Header.Set("X-Forwarded-For", forwardedFor) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("login request: %v", err) + } + defer resp.Body.Close() + return resp.StatusCode +} + +func countThrottled(t *testing.T, baseURL string, attempts int, forwardedFor func(i int) string) int { + t.Helper() + throttled := 0 + for i := 0; i < attempts; i++ { + xff := "" + if forwardedFor != nil { + xff = forwardedFor(i) + } + if badLogin(t, baseURL, xff) == http.StatusTooManyRequests { + throttled++ + } + } + return throttled +} + +// TestRotatingForwardedForCannotBypassTheLimiterEndToEnd is the wire-level guard +// for #1377: with no trusted proxies configured the limiter must key on the real +// TCP peer, so rotating the header cannot buy a fresh quota. +func TestRotatingForwardedForCannotBypassTheLimiterEndToEnd(t *testing.T) { + e := newEnv(t) + defer e.teardown() + + inst := e.start(t) + defer inst.stop() + + throttled := countThrottled(t, inst.baseURL, 40, func(i int) string { + return fmt.Sprintf("203.0.113.%d", i%250+1) + }) + if throttled == 0 { + t.Fatal("rotating X-Forwarded-For bypassed the login rate limiter entirely") + } + t.Logf("rotating header: %d/40 throttled", throttled) +} + +// TestWhitespacePaddedForwardedForCannotBypassTheLimiterEndToEnd covers the +// padding variant, which the old hand-rolled parser also treated as distinct. +func TestWhitespacePaddedForwardedForCannotBypassTheLimiterEndToEnd(t *testing.T) { + e := newEnv(t) + defer e.teardown() + + inst := e.start(t) + defer inst.stop() + + throttled := countThrottled(t, inst.baseURL, 40, func(i int) string { + return strings.Repeat(" ", i%8) + "203.0.113.9" + }) + if throttled == 0 { + t.Fatal("whitespace-padded X-Forwarded-For bypassed the login rate limiter") + } + t.Logf("padded header: %d/40 throttled", throttled) +} + +// TestTrustedProxyForwardedForIsHonouredEndToEnd is the counterpart: once the +// operator declares the proxy, distinct clients behind it get distinct buckets. +// A fix that simply ignored the header would fail this. +func TestTrustedProxyForwardedForIsHonouredEndToEnd(t *testing.T) { + e := newEnv(t) + defer e.teardown() + + inst := e.start(t, "CUBE_OPS_TRUSTED_PROXIES=127.0.0.1,::1") + defer inst.stop() + + noisy := countThrottled(t, inst.baseURL, 40, func(int) string { return "198.51.100.7" }) + if noisy == 0 { + t.Fatal("a client behind a trusted proxy was never throttled") + } + + quiet := countThrottled(t, inst.baseURL, 3, func(int) string { return "198.51.100.8" }) + if quiet != 0 { + t.Fatalf("a different client behind the same trusted proxy was throttled %d/3 times", quiet) + } + t.Logf("trusted proxy: noisy %d/40 throttled, quiet %d/3 throttled", noisy, quiet) +} + +// TestTrustAllProxiesIsRejectedAtStartupEndToEnd proves the config guard aborts +// the process rather than silently restoring the pre-fix behaviour. +func TestTrustAllProxiesIsRejectedAtStartupEndToEnd(t *testing.T) { + e := newEnv(t) + defer e.teardown() + + warm := e.start(t) + warm.stop() + + out := e.startExpectingExit(t, "CUBE_OPS_TRUSTED_PROXIES=0.0.0.0/0") + if !strings.Contains(out, "trusts every source address") { + t.Fatalf("0.0.0.0/0 was not rejected with the expected message:\n%s", tailLines(out, 15)) + } +} diff --git a/CubeOps/internal/auth/ratelimit.go b/CubeOps/internal/auth/ratelimit.go index 510c3871c..d8d265563 100644 --- a/CubeOps/internal/auth/ratelimit.go +++ b/CubeOps/internal/auth/ratelimit.go @@ -5,6 +5,7 @@ package auth import ( "net/http" + "sort" "sync" "time" @@ -26,6 +27,11 @@ type loginLimiter struct { window time.Duration } +const ( + sweepThreshold = 4096 + evictTarget = sweepThreshold / 2 +) + var defaultLoginLimiter = &loginLimiter{ failures: make(map[string][]time.Time), limit: 5, // 5 failed attempts @@ -41,6 +47,12 @@ func (l *loginLimiter) recordFailure(ip string) { defer l.mu.Unlock() now := time.Now() cutoff := now.Add(-l.window) + if len(l.failures) >= sweepThreshold { + l.sweepLocked(cutoff) + if len(l.failures) >= sweepThreshold { + l.evictOldestLocked(evictTarget) + } + } fails := l.failures[ip] // Drop expired entries. kept := fails[:0] @@ -57,6 +69,44 @@ func (l *loginLimiter) recordFailure(ip string) { } } +func (l *loginLimiter) sweepLocked(cutoff time.Time) { + for ip, fails := range l.failures { + kept := fails[:0] + for _, t := range fails { + if t.After(cutoff) { + kept = append(kept, t) + } + } + if len(kept) == 0 { + delete(l.failures, ip) + } else { + l.failures[ip] = kept + } + } +} + +func (l *loginLimiter) evictOldestLocked(target int) { + type entry struct { + ip string + last time.Time + } + entries := make([]entry, 0, len(l.failures)) + for ip, fails := range l.failures { + if len(fails) == 0 { + delete(l.failures, ip) + continue + } + entries = append(entries, entry{ip: ip, last: fails[len(fails)-1]}) + } + if len(entries) <= target { + return + } + sort.Slice(entries, func(i, j int) bool { return entries[i].last.Before(entries[j].last) }) + for i := 0; i < len(entries)-target; i++ { + delete(l.failures, entries[i].ip) + } +} + // isBlocked reports whether the IP has exceeded the failure limit. // It also prunes expired entries and deletes the map entry when empty, // so read-only checks also contribute to memory hygiene. @@ -82,26 +132,11 @@ func (l *loginLimiter) isBlocked(ip string) bool { return count >= l.limit } -// clientIP extracts the client IP from the request, honoring -// X-Forwarded-For (set by nginx). Falls back to RemoteAddr. -func clientIP(c *gin.Context) string { - if xff := c.GetHeader("X-Forwarded-For"); xff != "" { - // Use the first (leftmost) address — that is the original client. - for i := 0; i < len(xff); i++ { - if xff[i] == ',' { - return xff[:i] - } - } - return xff - } - return c.ClientIP() -} - // LoginRateLimit is a gin middleware that blocks IPs with too many recent // failed login attempts. It must be installed only on the /auth/login route. func LoginRateLimit() gin.HandlerFunc { return func(c *gin.Context) { - ip := clientIP(c) + ip := c.ClientIP() if defaultLoginLimiter.isBlocked(ip) { logging.G(c.Request.Context()).Warnf("login rate limit triggered: client_ip=%s path=%s", ip, c.Request.URL.Path) c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{ @@ -116,5 +151,5 @@ func LoginRateLimit() gin.HandlerFunc { // markLoginFailure is called by the Login handler when authentication fails. // It is exported so the handler can trigger it after a failed login. func markLoginFailure(c *gin.Context) { - defaultLoginLimiter.recordFailure(clientIP(c)) + defaultLoginLimiter.recordFailure(c.ClientIP()) } diff --git a/CubeOps/internal/auth/ratelimit_xff_test.go b/CubeOps/internal/auth/ratelimit_xff_test.go new file mode 100644 index 000000000..e6b9ae50c --- /dev/null +++ b/CubeOps/internal/auth/ratelimit_xff_test.go @@ -0,0 +1,216 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package auth + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" +) + +func newLimiterRouter(t *testing.T, trustedProxies []string) *gin.Engine { + t.Helper() + gin.SetMode(gin.TestMode) + r := gin.New() + if err := r.SetTrustedProxies(trustedProxies); err != nil { + t.Fatalf("SetTrustedProxies: %v", err) + } + r.POST("/login", LoginRateLimit(), func(c *gin.Context) { + markLoginFailure(c) + c.Status(http.StatusUnauthorized) + }) + return r +} + +func attempt(r *gin.Engine, peer string, xff string) int { + req := httptest.NewRequest(http.MethodPost, "/login", nil) + req.RemoteAddr = peer + if xff != "" { + req.Header.Set("X-Forwarded-For", xff) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + return w.Code +} + +func countBlocked(r *gin.Engine, peer string, xffs []string) int { + blocked := 0 + for _, xff := range xffs { + if attempt(r, peer, xff) == http.StatusTooManyRequests { + blocked++ + } + } + return blocked +} + +func resetLimiter(t *testing.T) { + t.Helper() + defaultLoginLimiter.mu.Lock() + defaultLoginLimiter.failures = map[string][]time.Time{} + defaultLoginLimiter.mu.Unlock() +} + +func TestRotatingForwardedForCannotBypassTheLimiter(t *testing.T) { + resetLimiter(t) + r := newLimiterRouter(t, nil) + + xffs := make([]string, 0, 40) + for i := 0; i < 40; i++ { + xffs = append(xffs, "10.1.1."+string(rune('0'+i%10))+"9") + } + + blocked := countBlocked(r, "203.0.113.9:1234", xffs) + if blocked == 0 { + t.Fatal("rotating X-Forwarded-For bypassed the limiter entirely") + } + if blocked < 30 { + t.Fatalf("only %d/40 attempts were blocked; the peer IP is not being used as the key", blocked) + } +} + +func TestWhitespacePaddedForwardedForCannotBypassTheLimiter(t *testing.T) { + resetLimiter(t) + r := newLimiterRouter(t, nil) + + xffs := make([]string, 0, 20) + for i := 0; i < 20; i++ { + pad := "" + for j := 0; j < i; j++ { + pad += " " + } + xffs = append(xffs, pad+"198.51.100.7") + } + + if blocked := countBlocked(r, "203.0.113.9:1234", xffs); blocked == 0 { + t.Fatal("whitespace-padded X-Forwarded-For bypassed the limiter") + } +} + +func TestLimiterStillFiresWithoutForwardedFor(t *testing.T) { + resetLimiter(t) + r := newLimiterRouter(t, nil) + + blocked := 0 + for i := 0; i < 10; i++ { + if attempt(r, "203.0.113.9:1234", "") == http.StatusTooManyRequests { + blocked++ + } + } + if blocked == 0 { + t.Fatal("baseline broken: the limiter never fired") + } +} + +func TestDistinctPeerIPsGetIndependentBuckets(t *testing.T) { + resetLimiter(t) + r := newLimiterRouter(t, nil) + + for i := 0; i < 6; i++ { + attempt(r, "203.0.113.9:1234", "") + } + if code := attempt(r, "203.0.113.9:1234", ""); code != http.StatusTooManyRequests { + t.Fatalf("first peer should be blocked, got %d", code) + } + if code := attempt(r, "203.0.113.10:1234", ""); code == http.StatusTooManyRequests { + t.Fatal("a different peer IP was blocked by another peer's failures") + } +} + +func TestTrustedProxyForwardedForIsHonoured(t *testing.T) { + resetLimiter(t) + r := newLimiterRouter(t, []string{"203.0.113.9"}) + + for i := 0; i < 6; i++ { + attempt(r, "203.0.113.9:1234", "198.51.100.7") + } + if code := attempt(r, "203.0.113.9:1234", "198.51.100.7"); code != http.StatusTooManyRequests { + t.Fatalf("a client behind a trusted proxy should be blocked, got %d", code) + } + if code := attempt(r, "203.0.113.9:1234", "198.51.100.8"); code == http.StatusTooManyRequests { + t.Fatal("a different client behind the same trusted proxy was blocked") + } +} + +func distinctIP(i int) string { + return fmt.Sprintf("10.%d.%d.%d", (i>>16)&0xff, (i>>8)&0xff, i&0xff) +} + +func mapSize(l *loginLimiter) int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.failures) +} + +func TestExpiredEntriesAreSweptOnceTheThresholdIsReached(t *testing.T) { + l := &loginLimiter{failures: map[string][]time.Time{}, limit: 5, window: 20 * time.Millisecond} + for i := 0; i < sweepThreshold; i++ { + l.recordFailure(distinctIP(i)) + } + if got := mapSize(l); got != sweepThreshold { + t.Fatalf("map holds %d entries before the sweep, want %d", got, sweepThreshold) + } + + time.Sleep(40 * time.Millisecond) + l.recordFailure(distinctIP(sweepThreshold)) + + if got := mapSize(l); got != 1 { + t.Fatalf("map holds %d entries after the sweep, want only the fresh one", got) + } +} + +func TestLiveEntriesAreEvictedSoTheMapStaysBounded(t *testing.T) { + l := &loginLimiter{failures: map[string][]time.Time{}, limit: 5, window: time.Hour} + const total = sweepThreshold * 3 + low := total + for i := 0; i < total; i++ { + l.recordFailure(distinctIP(i)) + got := mapSize(l) + if got > sweepThreshold { + t.Fatalf("map grew to %d entries at i=%d, above the threshold %d", got, i, sweepThreshold) + } + if i >= total-sweepThreshold && got < low { + low = got + } + } + if low > evictTarget+1 { + t.Fatalf("map never fell below %d entries, so live entries are never evicted", low) + } +} + +func TestEvictionKeepsTheMostRecentAttackers(t *testing.T) { + l := &loginLimiter{failures: map[string][]time.Time{}, limit: 5, window: time.Hour} + + filler := 0 + fillTo := func(target int) { + for mapSize(l) < target { + l.recordFailure(distinctIP(filler)) + filler++ + } + } + + fillTo(sweepThreshold - 8) + + recent := "203.0.113.7" + for i := 0; i < l.limit; i++ { + l.recordFailure(recent) + } + + fillTo(sweepThreshold) + if got := mapSize(l); got != sweepThreshold { + t.Fatalf("map holds %d entries, want exactly %d so the next failure triggers eviction", got, sweepThreshold) + } + + l.recordFailure(distinctIP(filler)) + + if got := mapSize(l); got > evictTarget+1 { + t.Fatalf("eviction did not run: map still holds %d entries", got) + } + if !l.isBlocked(recent) { + t.Fatal("the most recently active IP was evicted, so its failures were forgotten") + } +} diff --git a/CubeOps/internal/config/config.go b/CubeOps/internal/config/config.go index 17337df2d..2c708ad2d 100644 --- a/CubeOps/internal/config/config.go +++ b/CubeOps/internal/config/config.go @@ -21,6 +21,7 @@ package config import ( "fmt" + "net" "net/url" "os" "strconv" @@ -34,12 +35,13 @@ import ( // Config holds all CubeOps runtime configuration. type Config struct { // Server - Bind string `yaml:"bind"` - LogLevel string `yaml:"log_level"` - LogDir string `yaml:"log_dir"` - LogFileNum int `yaml:"log_file_num"` - LogFileSize int `yaml:"log_file_size"` - JWTSecret string `yaml:"jwt_secret"` + Bind string `yaml:"bind"` + LogLevel string `yaml:"log_level"` + LogDir string `yaml:"log_dir"` + LogFileNum int `yaml:"log_file_num"` + LogFileSize int `yaml:"log_file_size"` + JWTSecret string `yaml:"jwt_secret"` + TrustedProxies []string `yaml:"trusted_proxies"` // Database — either a single URL or the individual fields below. DatabaseURL string `yaml:"database_url"` @@ -114,6 +116,10 @@ func Load() (*Config, error) { cfg.SandboxDomain = "cube.app" } + if err := validateTrustedProxies(cfg.TrustedProxies); err != nil { + return nil, err + } + // JWT_SECRET is optional — if not set, it will be auto-generated and // persisted to the DB on first startup (see store.bootstrapJWTSecret). if cfg.DatabaseURL == "" { @@ -281,6 +287,9 @@ func overrideFromEnv(cfg *Config) { if v := os.Getenv("JWT_SECRET"); v != "" { cfg.JWTSecret = v } + if v := os.Getenv("CUBE_OPS_TRUSTED_PROXIES"); v != "" { + cfg.TrustedProxies = splitAndTrim(v) + } if v := os.Getenv("DATABASE_URL"); v != "" { cfg.DatabaseURL = v } @@ -325,3 +334,58 @@ func overrideFromEnv(cfg *Config) { } } } + +// BroadTrustedProxies returns the configured entries whose prefix is wide +// enough that a large share of the network can assert a client IP. Callers +// warn about these; they are not rejected because a pod or VPC CIDR can +// legitimately be this wide. +func BroadTrustedProxies(entries []string) []string { + var broad []string + for _, e := range entries { + _, ipNet, err := net.ParseCIDR(strings.TrimSpace(e)) + if err != nil { + continue + } + ones, bits := ipNet.Mask.Size() + limit := 16 + if bits > 32 { + limit = 64 + } + if ones < limit { + broad = append(broad, e) + } + } + return broad +} + +func validateTrustedProxies(entries []string) error { + for i, e := range entries { + e = strings.TrimSpace(e) + entries[i] = e + if _, ipNet, err := net.ParseCIDR(e); err == nil { + ones, bits := ipNet.Mask.Size() + if ones == 0 && bits != 0 { + return fmt.Errorf("trusted_proxies entry %q trusts every source address, which disables client-IP validation entirely; list the proxy addresses or CIDR blocks instead (trusted_proxies in YAML %s, or CUBE_OPS_TRUSTED_PROXIES)", + e, yamlConfigPath()) + } + continue + } + if net.ParseIP(e) != nil { + continue + } + return fmt.Errorf("trusted_proxies entry %q is not an IP address or CIDR block; use forms like 10.1.2.3, 10.1.0.0/16 or ::1 (trusted_proxies in YAML %s, or CUBE_OPS_TRUSTED_PROXIES). Wildcards such as \"*\", 0.0.0.0/0 and ::/0 are not accepted", + e, yamlConfigPath()) + } + return nil +} + +func splitAndTrim(v string) []string { + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} diff --git a/CubeOps/internal/config/config_trusted_proxies_test.go b/CubeOps/internal/config/config_trusted_proxies_test.go new file mode 100644 index 000000000..4dd0b65c4 --- /dev/null +++ b/CubeOps/internal/config/config_trusted_proxies_test.go @@ -0,0 +1,76 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "strings" + "testing" +) + +func TestValidateTrustedProxiesAcceptsSpecificProxies(t *testing.T) { + for _, entries := range [][]string{ + nil, + {}, + {"127.0.0.1"}, + {"::1"}, + {"10.42.0.7", "10.42.0.8"}, + {"10.42.0.0/16"}, + {"fd00::/64"}, + } { + if err := validateTrustedProxies(entries); err != nil { + t.Errorf("validateTrustedProxies(%v) = %v, want nil", entries, err) + } + } +} + +func TestValidateTrustedProxiesRejectsTrustAll(t *testing.T) { + for _, entry := range []string{"0.0.0.0/0", "::/0", "*"} { + err := validateTrustedProxies([]string{entry}) + if err == nil { + t.Errorf("validateTrustedProxies(%q) = nil, want an error", entry) + continue + } + if !strings.Contains(err.Error(), entry) { + t.Errorf("error for %q does not name the entry: %v", entry, err) + } + } +} + +func TestValidateTrustedProxiesRejectsMalformedEntries(t *testing.T) { + for _, entry := range []string{"not-an-ip", "10.0.0.1/33", "example.com", "10.0.0.256"} { + if err := validateTrustedProxies([]string{entry}); err == nil { + t.Errorf("validateTrustedProxies(%q) = nil, want an error", entry) + } + } +} + +func TestValidateTrustedProxiesTrimsEntries(t *testing.T) { + entries := []string{" 10.0.0.1 ", "\t10.42.0.0/16\n"} + if err := validateTrustedProxies(entries); err != nil { + t.Fatalf("validateTrustedProxies rejected padded entries: %v", err) + } + if entries[0] != "10.0.0.1" || entries[1] != "10.42.0.0/16" { + t.Fatalf("entries were not trimmed in place: %q", entries) + } +} + +func TestBroadTrustedProxiesFlagsWideRanges(t *testing.T) { + broad := BroadTrustedProxies([]string{ + "10.0.0.0/8", + "172.16.0.0/12", + "10.42.0.0/16", + "10.42.1.5", + "fd00::/48", + "fd00::/64", + }) + want := map[string]bool{"10.0.0.0/8": true, "172.16.0.0/12": true, "fd00::/48": true} + if len(broad) != len(want) { + t.Fatalf("BroadTrustedProxies = %v, want %d entries", broad, len(want)) + } + for _, b := range broad { + if !want[b] { + t.Errorf("BroadTrustedProxies flagged %q, which is narrow enough", b) + } + } +} diff --git a/CubeOps/internal/server/server.go b/CubeOps/internal/server/server.go index 6c274ea7e..f1d10f870 100644 --- a/CubeOps/internal/server/server.go +++ b/CubeOps/internal/server/server.go @@ -81,6 +81,19 @@ func (s *Server) buildRouter() *gin.Engine { // to stdout and bypasses any logger the operator has configured. gin.SetMode(gin.ReleaseMode) r := gin.New() + if err := r.SetTrustedProxies(s.cfg.TrustedProxies); err != nil { + logging.G(context.Background()).Errorf("invalid trusted_proxies, falling back to trusting none: err=%q", err.Error()) + _ = r.SetTrustedProxies(nil) + } + if len(s.cfg.TrustedProxies) == 0 { + logging.G(context.Background()).Warnf("trusted_proxies is empty: X-Forwarded-For is ignored and the login rate limiter keys on the TCP peer. " + + "Behind a reverse proxy every client shares one bucket, so five failures from anyone throttle logins for that proxy. " + + "Set trusted_proxies (or CUBE_OPS_TRUSTED_PROXIES) to the proxy address or CIDR.") + } + if broad := config.BroadTrustedProxies(s.cfg.TrustedProxies); len(broad) > 0 { + logging.G(context.Background()).Warnf("trusted_proxies contains broad ranges %v: every host in them can assert any client IP via X-Forwarded-For "+ + "and so bypass the login rate limiter. Narrow this to the actual proxy addresses.", broad) + } r.Use(requestLogger()) r.Use(cubeopsRecovery()) diff --git a/deploy/kubernetes/chart/templates/ops.yaml b/deploy/kubernetes/chart/templates/ops.yaml index 95b832bf7..12f9b1df5 100644 --- a/deploy/kubernetes/chart/templates/ops.yaml +++ b/deploy/kubernetes/chart/templates/ops.yaml @@ -37,6 +37,10 @@ spec: {{- include "cube.timezoneEnv" . | nindent 12 }} - name: CUBE_OPS_BIND value: {{ .Values.cubeOps.bind | quote }} + {{- with .Values.cubeOps.trustedProxies }} + - name: CUBE_OPS_TRUSTED_PROXIES + value: {{ join "," . | quote }} + {{- end }} - name: CUBE_MASTER_ADDR value: {{ printf "http://%s" (include "cube.masterEndpoint" .) | quote }} {{- if eq (include "cube.dbDriver" .) "postgres" }} diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 9c6159f3c..c96bfa24b 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -339,6 +339,15 @@ cubeOps: replicas: 1 podAnnotations: {} bind: "0.0.0.0:3010" + # Reverse proxies allowed to assert the client IP via X-Forwarded-For. + # Empty trusts none: the login rate limiter keys on the real TCP peer, which + # is the safe default. CubeOps is a ClusterIP service, so anything that can + # reach it -- including tenant sandbox pods -- would be trusted if listed + # here; do NOT use a broad private range. Set this to the specific ingress or + # webui pod CIDR, and prefer a NetworkPolicy that stops tenant pods reaching + # the ops service at all. While it is empty, all clients behind a proxy share + # one bucket, so five failures from anyone throttles logins for that proxy. + trustedProxies: [] # Empty uses cubeProxy.domain for CUBE_API_SANDBOX_DOMAIN. sandboxDomain: "" service: diff --git a/deploy/one-click/env.example b/deploy/one-click/env.example index 7b5e3f391..e3a2e0939 100644 --- a/deploy/one-click/env.example +++ b/deploy/one-click/env.example @@ -251,6 +251,12 @@ CUBE_API_SANDBOX_DOMAIN=cube.app # Must bind 0.0.0.0 in All-in-One mode so the WebUI nginx container can # reach CubeOps via host.docker.internal:3010. CUBE_OPS_BIND=0.0.0.0:3010 +# Reverse proxies allowed to assert the client IP via X-Forwarded-For. +# Empty trusts none, which is the safe default: the login rate limiter keys on +# the real TCP peer. Do NOT put the whole Docker bridge range here -- sandbox +# containers share it and could then spoof the header. Set it to the specific +# address the WebUI nginx connects from if you want per-client login buckets. +CUBE_OPS_TRUSTED_PROXIES= CUBE_OPS_LOG_LEVEL=info CUBE_OPS_LOG_DIR=/data/log/CubeOps CUBE_OPS_UPSTREAM=http://host.docker.internal:3010 diff --git a/deploy/one-click/scripts/systemd/cubeops-start.sh b/deploy/one-click/scripts/systemd/cubeops-start.sh index a8f6bcffe..3f63190cb 100644 --- a/deploy/one-click/scripts/systemd/cubeops-start.sh +++ b/deploy/one-click/scripts/systemd/cubeops-start.sh @@ -19,6 +19,7 @@ mkdir -p "${CUBE_OPS_LOG_DIR}" # Bind address — must be 0.0.0.0 in All-in-One mode so the WebUI nginx # container can reach CubeOps via host.docker.internal:3010. export CUBE_OPS_BIND="${CUBE_OPS_BIND:-0.0.0.0:3010}" +export CUBE_OPS_TRUSTED_PROXIES="${CUBE_OPS_TRUSTED_PROXIES:-}" export CUBE_OPS_LOG_LEVEL="${CUBE_OPS_LOG_LEVEL:-info}" # CubeMaster address (same host in All-in-One mode).