From dd53d613bc65a5eb987e4132ef6627b15a3871ce Mon Sep 17 00:00:00 2001 From: Choi-Eunseok Date: Fri, 5 Jun 2026 14:09:15 +0900 Subject: [PATCH] fix(app-gateway): cache resolved fixed IPs --- .github/workflows/deploy-app-gateway.yml | 3 +- cmd/app-gateway/config.go | 6 ++ cmd/app-gateway/config_test.go | 23 +++-- cmd/app-gateway/main.go | 2 +- cmd/app-gateway/resolver_cache.go | 52 +++++++++++ cmd/app-gateway/resolver_cache_test.go | 105 +++++++++++++++++++++++ deploy/systemd/app-gateway.env.example | 3 + 7 files changed, 184 insertions(+), 10 deletions(-) create mode 100644 cmd/app-gateway/resolver_cache.go create mode 100644 cmd/app-gateway/resolver_cache_test.go diff --git a/.github/workflows/deploy-app-gateway.yml b/.github/workflows/deploy-app-gateway.yml index 58d8e92..70b0f7a 100644 --- a/.github/workflows/deploy-app-gateway.yml +++ b/.github/workflows/deploy-app-gateway.yml @@ -62,7 +62,7 @@ jobs: - name: Write app-gateway env file uses: appleboy/ssh-action@v0.1.10 env: - ENV_LIST: OS_AUTH_URL,OS_USERNAME,OS_PASSWORD,OS_PROJECT_NAME,OS_USER_DOMAIN_NAME,CF_ACCESS_CLIENT_ID,CF_ACCESS_CLIENT_SECRET,DB_DRIVER,DB_DSN,APP_GATEWAY_PORT,RCP_NS_PROXY_SOCK,RCP_APP_GW_FIXED_NETWORK,RCP_APP_GW_READ_TIMEOUT,RCP_APP_GW_SHUTDOWN_TIMEOUT,RCP_APP_GW_LOG_LEVEL + ENV_LIST: OS_AUTH_URL,OS_USERNAME,OS_PASSWORD,OS_PROJECT_NAME,OS_USER_DOMAIN_NAME,CF_ACCESS_CLIENT_ID,CF_ACCESS_CLIENT_SECRET,DB_DRIVER,DB_DSN,APP_GATEWAY_PORT,RCP_NS_PROXY_SOCK,RCP_APP_GW_FIXED_NETWORK,RCP_APP_GW_READ_TIMEOUT,RCP_APP_GW_SHUTDOWN_TIMEOUT,RCP_APP_GW_FIXED_IP_CACHE_TTL,RCP_APP_GW_LOG_LEVEL OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} OS_USERNAME: ${{ secrets.OS_USERNAME }} OS_PASSWORD: ${{ secrets.OS_PASSWORD }} @@ -77,6 +77,7 @@ jobs: RCP_APP_GW_FIXED_NETWORK: ${{ secrets.RCP_APP_GW_FIXED_NETWORK }} RCP_APP_GW_READ_TIMEOUT: ${{ secrets.RCP_APP_GW_READ_TIMEOUT || '30s' }} RCP_APP_GW_SHUTDOWN_TIMEOUT: ${{ secrets.RCP_APP_GW_SHUTDOWN_TIMEOUT || '30s' }} + RCP_APP_GW_FIXED_IP_CACHE_TTL: ${{ secrets.RCP_APP_GW_FIXED_IP_CACHE_TTL || '5m' }} RCP_APP_GW_LOG_LEVEL: ${{ secrets.RCP_APP_GW_LOG_LEVEL || 'info' }} with: host: ${{ secrets.SERVER_HOST }} diff --git a/cmd/app-gateway/config.go b/cmd/app-gateway/config.go index 2f47a0f..2e12e76 100644 --- a/cmd/app-gateway/config.go +++ b/cmd/app-gateway/config.go @@ -16,6 +16,7 @@ type Config struct { DBDSN string ReadTimeout time.Duration ShutdownTimeout time.Duration + FixedIPCacheTTL time.Duration LogLevel string } @@ -54,6 +55,10 @@ func LoadConfig(getenv func(string) string) (*Config, error) { if err != nil { return nil, err } + fixedIPCacheTTL, err := utils.EnvPositiveDuration(getenv, "RCP_APP_GW_FIXED_IP_CACHE_TTL", 5*time.Minute) + if err != nil { + return nil, err + } logLevel, err := utils.EnvLogLevel(getenv, "RCP_APP_GW_LOG_LEVEL", "info") if err != nil { return nil, err @@ -67,6 +72,7 @@ func LoadConfig(getenv func(string) string) (*Config, error) { DBDSN: dbDSN, ReadTimeout: readTimeout, ShutdownTimeout: shutdownTimeout, + FixedIPCacheTTL: fixedIPCacheTTL, LogLevel: logLevel, }, nil } diff --git a/cmd/app-gateway/config_test.go b/cmd/app-gateway/config_test.go index 85573ed..1566459 100644 --- a/cmd/app-gateway/config_test.go +++ b/cmd/app-gateway/config_test.go @@ -32,6 +32,9 @@ func TestLoadConfigDefaults(t *testing.T) { if cfg.ShutdownTimeout != 30*time.Second { t.Fatalf("ShutdownTimeout got %v", cfg.ShutdownTimeout) } + if cfg.FixedIPCacheTTL != 5*time.Minute { + t.Fatalf("FixedIPCacheTTL got %v", cfg.FixedIPCacheTTL) + } if cfg.LogLevel != "info" { t.Fatalf("LogLevel got %q", cfg.LogLevel) } @@ -39,14 +42,15 @@ func TestLoadConfigDefaults(t *testing.T) { func TestLoadConfigUsesExplicitValues(t *testing.T) { cfg, err := LoadConfig(envFromMap(map[string]string{ - "APP_GATEWAY_PORT": "19090", - "RCP_NS_PROXY_SOCK": "/tmp/ns-proxy.sock", - "DB_DRIVER": "postgres", - "DB_DSN": "host=db dbname=rcp", - "RCP_APP_GW_FIXED_NETWORK": "tenant-net", - "RCP_APP_GW_READ_TIMEOUT": "5s", - "RCP_APP_GW_SHUTDOWN_TIMEOUT": "7s", - "RCP_APP_GW_LOG_LEVEL": "debug", + "APP_GATEWAY_PORT": "19090", + "RCP_NS_PROXY_SOCK": "/tmp/ns-proxy.sock", + "DB_DRIVER": "postgres", + "DB_DSN": "host=db dbname=rcp", + "RCP_APP_GW_FIXED_NETWORK": "tenant-net", + "RCP_APP_GW_READ_TIMEOUT": "5s", + "RCP_APP_GW_SHUTDOWN_TIMEOUT": "7s", + "RCP_APP_GW_FIXED_IP_CACHE_TTL": "9s", + "RCP_APP_GW_LOG_LEVEL": "debug", })) if err != nil { t.Fatalf("unexpected err: %v", err) @@ -63,6 +67,9 @@ func TestLoadConfigUsesExplicitValues(t *testing.T) { if cfg.ReadTimeout != 5*time.Second || cfg.ShutdownTimeout != 7*time.Second { t.Fatalf("unexpected timeouts: %+v", cfg) } + if cfg.FixedIPCacheTTL != 9*time.Second { + t.Fatalf("FixedIPCacheTTL got %v", cfg.FixedIPCacheTTL) + } if cfg.LogLevel != "debug" { t.Fatalf("LogLevel got %q", cfg.LogLevel) } diff --git a/cmd/app-gateway/main.go b/cmd/app-gateway/main.go index c860d50..1d43165 100644 --- a/cmd/app-gateway/main.go +++ b/cmd/app-gateway/main.go @@ -52,7 +52,7 @@ func main() { fatal("openstack compute client: %v", err) } - app := NewServer(cfg, log, newRepo(db), resolver) + app := NewServer(cfg, log, newRepo(db), newCachedFixedIPResolver(resolver, cfg.FixedIPCacheTTL)) httpSrv := &http.Server{ Addr: cfg.Listen, Handler: app, diff --git a/cmd/app-gateway/resolver_cache.go b/cmd/app-gateway/resolver_cache.go new file mode 100644 index 0000000..56e21be --- /dev/null +++ b/cmd/app-gateway/resolver_cache.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "sync" + "time" +) + +type cachedFixedIPResolver struct { + next fixedIPResolver + ttl time.Duration + now func() time.Time + mu sync.RWMutex + cache map[string]fixedIPCacheEntry +} + +type fixedIPCacheEntry struct { + ip string + expiresAt time.Time +} + +func newCachedFixedIPResolver(next fixedIPResolver, ttl time.Duration) *cachedFixedIPResolver { + return &cachedFixedIPResolver{ + next: next, + ttl: ttl, + now: time.Now, + cache: make(map[string]fixedIPCacheEntry), + } +} + +func (c *cachedFixedIPResolver) ResolveFixedIPv4(ctx context.Context, openstackID string) (string, error) { + now := c.now() + c.mu.RLock() + entry, ok := c.cache[openstackID] + c.mu.RUnlock() + if ok && now.Before(entry.expiresAt) { + return entry.ip, nil + } + + ip, err := c.next.ResolveFixedIPv4(ctx, openstackID) + if err != nil { + if ok && entry.ip != "" { + return entry.ip, nil + } + return "", err + } + + c.mu.Lock() + c.cache[openstackID] = fixedIPCacheEntry{ip: ip, expiresAt: now.Add(c.ttl)} + c.mu.Unlock() + return ip, nil +} diff --git a/cmd/app-gateway/resolver_cache_test.go b/cmd/app-gateway/resolver_cache_test.go new file mode 100644 index 0000000..8516189 --- /dev/null +++ b/cmd/app-gateway/resolver_cache_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" +) + +type countingResolver struct { + ip string + err error + calls int +} + +func (c *countingResolver) ResolveFixedIPv4(context.Context, string) (string, error) { + c.calls++ + if c.err != nil { + return "", c.err + } + return c.ip, nil +} + +func TestCachedFixedIPResolverCachesSuccessfulLookup(t *testing.T) { + next := &countingResolver{ip: "10.0.0.8"} + resolver := newCachedFixedIPResolver(next, time.Minute) + + for range 2 { + got, err := resolver.ResolveFixedIPv4(context.Background(), "os-1") + if err != nil { + t.Fatalf("ResolveFixedIPv4 returned error: %v", err) + } + if got != "10.0.0.8" { + t.Fatalf("ip got %q", got) + } + } + + if next.calls != 1 { + t.Fatalf("underlying resolver calls got %d", next.calls) + } +} + +func TestCachedFixedIPResolverDoesNotCacheErrors(t *testing.T) { + next := &countingResolver{err: errors.New("nova timeout")} + resolver := newCachedFixedIPResolver(next, time.Minute) + + for range 2 { + _, err := resolver.ResolveFixedIPv4(context.Background(), "os-1") + if err == nil { + t.Fatalf("expected error") + } + } + + if next.calls != 2 { + t.Fatalf("underlying resolver calls got %d", next.calls) + } +} + +func TestCachedFixedIPResolverRefreshesAfterTTL(t *testing.T) { + next := &countingResolver{ip: "10.0.0.8"} + resolver := newCachedFixedIPResolver(next, time.Minute) + now := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) + resolver.now = func() time.Time { return now } + + if _, err := resolver.ResolveFixedIPv4(context.Background(), "os-1"); err != nil { + t.Fatalf("ResolveFixedIPv4 returned error: %v", err) + } + + next.ip = "10.0.0.7" + now = now.Add(time.Minute + time.Second) + got, err := resolver.ResolveFixedIPv4(context.Background(), "os-1") + if err != nil { + t.Fatalf("ResolveFixedIPv4 returned error: %v", err) + } + if got != "10.0.0.7" { + t.Fatalf("ip got %q", got) + } + if next.calls != 2 { + t.Fatalf("underlying resolver calls got %d", next.calls) + } +} + +func TestCachedFixedIPResolverUsesStaleIPOnRefreshError(t *testing.T) { + next := &countingResolver{ip: "10.0.0.8"} + resolver := newCachedFixedIPResolver(next, time.Minute) + now := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) + resolver.now = func() time.Time { return now } + + if _, err := resolver.ResolveFixedIPv4(context.Background(), "os-1"); err != nil { + t.Fatalf("ResolveFixedIPv4 returned error: %v", err) + } + + next.err = errors.New("nova timeout") + now = now.Add(time.Minute + time.Second) + got, err := resolver.ResolveFixedIPv4(context.Background(), "os-1") + if err != nil { + t.Fatalf("ResolveFixedIPv4 returned error: %v", err) + } + if got != "10.0.0.8" { + t.Fatalf("ip got %q", got) + } + if next.calls != 2 { + t.Fatalf("underlying resolver calls got %d", next.calls) + } +} diff --git a/deploy/systemd/app-gateway.env.example b/deploy/systemd/app-gateway.env.example index 8a7e802..6892bf4 100644 --- a/deploy/systemd/app-gateway.env.example +++ b/deploy/systemd/app-gateway.env.example @@ -12,6 +12,9 @@ DB_DSN=host=localhost port=5432 user=rcp password=secret dbname=rcp sslmode=disa # ns-proxy Unix socket 경로. RCP_NS_PROXY_SOCK=/run/rcp/ns-proxy.sock +# Nova fixed IP 조회 결과 캐시 TTL. 만료 후 조회 실패 시 마지막 성공 IP를 fallback으로 사용합니다. +RCP_APP_GW_FIXED_IP_CACHE_TTL=5m + # fixed IP 조회용 OpenStack 인증 설정. OS_AUTH_URL=https://openstack.example.com:5000/v3 OS_USERNAME=return