Skip to content

Commit 24d910b

Browse files
imrocimroc
committed
fix: 分页查询 CLB 信息，避免端口池超 20 个 CLB 时无法正常扩容
Signed-off-by: roc <[email protected]>
1 parent f36a050 commit 24d910b

File tree

10 files changed

+86
-79
lines changed

10 files changed

+86
-79
lines changed

‎pkg/clb/api.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"sigs.k8s.io/controller-runtime/pkg/log"
1010
)
1111

12-
func ApiCall[Req, Res any](ctx context.Context, apiName, region string, doReq func(ctx context.Context, client *clb.Client) (req Req, res Res, err error)) (res Res, err error) {
12+
func ApiCall[Req, Res any](ctx context.Context, writeOp bool, apiName, region string, doReq func(ctx context.Context, client *clb.Client) (req Req, res Res, err error)) (res Res, err error) {
1313
reqCount := 0
1414
start := time.Now()
1515
defer func() {
@@ -30,11 +30,11 @@ func ApiCall[Req, Res any](ctx context.Context, apiName, region string, doReq fu
3030
}
3131
before := time.Now()
3232
req, res, err := doReq(ctx, client)
33-
LogAPI(ctx, apiName, req, res, time.Since(before), err)
33+
LogAPI(ctx, writeOp, apiName, req, res, time.Since(before), err)
3434
reqCount++
3535
if err != nil {
3636
if IsRequestLimitExceededError(err) { // 云 API 限频，重试
37-
log.FromContext(ctx).V(3).Info(
37+
log.FromContext(ctx).Info(
3838
"clb api request limit exceeded",
3939
"api", apiName,
4040
"err", err,

‎pkg/clb/batch-listener.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ type listenerKey struct {
4747
const TkeListenerName = "TKE-LISTENER"
4848

4949
func doBatchCreateListener(apiName, region, lbId, protocol, certId, extensiveParameters string, tasks []*CreateListenerTask) (listenerIds []string, err error) {
50-
res, err := ApiCall(context.Background(), apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.CreateListenerRequest, res *clb.CreateListenerResponse, err error) {
50+
res, err := ApiCall(context.Background(), true, apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.CreateListenerRequest, res *clb.CreateListenerResponse, err error) {
5151
req = clb.NewCreateListenerRequest()
5252
req.LoadBalancerId = &lbId
5353
req.HealthCheck = &clb.HealthCheck{
@@ -154,7 +154,7 @@ var DescribeListenerChan = make(chan *DescribeListenerTask, 100)
154154
func startDescribeListenerProccessor(concurrent int) {
155155
apiName := "DescribeListeners"
156156
StartBatchProccessor(concurrent, apiName, false, DescribeListenerChan, func(region, lbId string, tasks []*DescribeListenerTask) {
157-
res, err := ApiCall(context.Background(), apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeListenersRequest, res *clb.DescribeListenersResponse, err error) {
157+
res, err := ApiCall(context.Background(), false, apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeListenersRequest, res *clb.DescribeListenersResponse, err error) {
158158
req = clb.NewDescribeListenersRequest()
159159
req.LoadBalancerId = &lbId
160160
for _, task := range tasks {
@@ -216,7 +216,7 @@ var (
216216
func startDeleteListenerProccessor(concurrent int) {
217217
apiName := "DeleteLoadBalancerListeners"
218218
StartBatchProccessor(concurrent, apiName, true, DeleteListenerChan, func(region, lbId string, tasks []*DeleteListenerTask) {
219-
res, err := ApiCall(context.Background(), apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.DeleteLoadBalancerListenersRequest, res *clb.DeleteLoadBalancerListenersResponse, err error) {
219+
res, err := ApiCall(context.Background(), true, apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.DeleteLoadBalancerListenersRequest, res *clb.DeleteLoadBalancerListenersResponse, err error) {
220220
req = clb.NewDeleteLoadBalancerListenersRequest()
221221
req.LoadBalancerId = &lbId
222222
for _, task := range tasks {

‎pkg/clb/batch-target.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ var RegisterTargetChan = make(chan *RegisterTargetTask, 100)
3030
func startRegisterTargetsProccessor(concurrent int) {
3131
apiName := "BatchRegisterTargets"
3232
StartBatchProccessor(concurrent, apiName, true, RegisterTargetChan, func(region, lbId string, tasks []*RegisterTargetTask) {
33-
res, err := ApiCall(context.Background(), apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.BatchRegisterTargetsRequest, res *clb.BatchRegisterTargetsResponse, err error) {
33+
res, err := ApiCall(context.Background(), true, apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.BatchRegisterTargetsRequest, res *clb.BatchRegisterTargetsResponse, err error) {
3434
req = clb.NewBatchRegisterTargetsRequest()
3535
req.LoadBalancerId = &lbId
3636
for _, task := range tasks {
@@ -89,7 +89,7 @@ var DescribeTargetsChan = make(chan *DescribeTargetsTask, 100)
8989
func startDescribeTargetsProccessor(concurrent int) {
9090
apiName := "DescribeTargets"
9191
StartBatchProccessor(concurrent, apiName, false, DescribeTargetsChan, func(region, lbId string, tasks []*DescribeTargetsTask) {
92-
res, err := ApiCall(context.Background(), apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeTargetsRequest, res *clb.DescribeTargetsResponse, err error) {
92+
res, err := ApiCall(context.Background(), false, apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeTargetsRequest, res *clb.DescribeTargetsResponse, err error) {
9393
req = clb.NewDescribeTargetsRequest()
9494
req.LoadBalancerId = &lbId
9595
for _, task := range tasks {
@@ -149,7 +149,7 @@ var DeregisterTargetsChan = make(chan *DeregisterTargetsTask, 100)
149149
func startDeregisterTargetsProccessor(concurrent int) {
150150
apiName := "BatchDeregisterTargets"
151151
StartBatchProccessor(concurrent, apiName, true, DeregisterTargetsChan, func(region, lbId string, tasks []*DeregisterTargetsTask) {
152-
res, err := ApiCall(context.Background(), apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.BatchDeregisterTargetsRequest, res *clb.BatchDeregisterTargetsResponse, err error) {
152+
res, err := ApiCall(context.Background(), true, apiName, region, func(ctx context.Context, client *clb.Client) (req *clb.BatchDeregisterTargetsRequest, res *clb.BatchDeregisterTargetsResponse, err error) {
153153
req = clb.NewBatchDeregisterTargetsRequest()
154154
req.LoadBalancerId = &lbId
155155
for _, task := range tasks {

‎pkg/clb/clb.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,15 @@ func GetClient(region string) *clb.Client {
3232
return client
3333
}
3434

35-
func LogAPI(ctx context.Context, apiName string, req, resp any, cost time.Duration, err error) {
35+
func LogAPI(ctx context.Context, writeOp bool, apiName string, req, resp any, cost time.Duration, err error) {
3636
var logger logr.Logger
3737
if ctx != nil {
3838
logger = log.FromContext(ctx)
3939
} else {
4040
logger = clbLog
4141
}
42+
if !writeOp {
43+
logger = logger.V(1)
44+
}
4245
logger.Info("CLB API Call", "api", apiName, "request", req, "response", resp, "cost", cost.String(), "error", err)
4346
}

‎pkg/clb/instance.go‎

Lines changed: 65 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func GetClb(ctx context.Context, lbId, region string) (instance *clb.LoadBalance
4343

4444
before := time.Now()
4545
resp, err := client.DescribeLoadBalancersWithContext(ctx, req)
46-
LogAPI(ctx, "DescribeLoadBalancers", req, resp, time.Since(before), err)
46+
LogAPI(ctx, false, "DescribeLoadBalancers", req, resp, time.Since(before), err)
4747
if err != nil {
4848
return
4949
}
@@ -81,7 +81,7 @@ func Create(ctx context.Context, region, vpcId, extensiveParameters string, num
8181
client := GetClient(region)
8282
before := time.Now()
8383
resp, err := client.CreateLoadBalancerWithContext(ctx, req)
84-
LogAPI(ctx, "CreateLoadBalancer", req, resp, time.Since(before), err)
84+
LogAPI(ctx, true, "CreateLoadBalancer", req, resp, time.Since(before), err)
8585
if err != nil {
8686
return
8787
}
@@ -121,7 +121,7 @@ func Delete(ctx context.Context, region string, lbIds ...string) error {
121121
client := GetClient(region)
122122
before := time.Now()
123123
resp, err := client.DeleteLoadBalancerWithContext(ctx, req)
124-
LogAPI(ctx, "DeleteLoadBalancer", req, resp, time.Since(before), err)
124+
LogAPI(ctx, true, "DeleteLoadBalancer", req, resp, time.Since(before), err)
125125
if err != nil {
126126
if IsLbIdNotFoundError(err) {
127127
if len(lbIds) == 1 { // lb 已全部删除，忽略
@@ -147,7 +147,7 @@ func CreateCLB(ctx context.Context, region string, req *clb.CreateLoadBalancerRe
147147
client := GetClient(region)
148148
before := time.Now()
149149
resp, err := client.CreateLoadBalancerWithContext(ctx, req)
150-
LogAPI(ctx, "CreateLoadBalancer", req, resp, time.Since(before), err)
150+
LogAPI(ctx, true, "CreateLoadBalancer", req, resp, time.Since(before), err)
151151
if err != nil {
152152
return
153153
}
@@ -171,57 +171,75 @@ type CLBInfo struct {
171171
}
172172

173173
func BatchGetClbInfo(ctx context.Context, lbIds []string, region string) (info map[string]*CLBInfo, err error) {
174-
res, err := ApiCall(context.Background(), "DescribeLoadBalancers", region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeLoadBalancersRequest, res *clb.DescribeLoadBalancersResponse, err error) {
175-
req = clb.NewDescribeLoadBalancersRequest()
176-
req.LoadBalancerIds = common.StringPtrs(lbIds)
177-
res, err = client.DescribeLoadBalancersWithContext(ctx, req)
178-
return
179-
})
180-
if err != nil {
181-
return nil, errors.WithStack(err)
182-
}
183-
if *res.Response.TotalCount == 0 || len(res.Response.LoadBalancerSet) == 0 {
184-
return
185-
}
186174
info = make(map[string]*CLBInfo)
187175
insIds := []*string{}
188-
for _, ins := range res.Response.LoadBalancerSet {
189-
lbInfo := &CLBInfo{
190-
LoadbalancerID: *ins.LoadBalancerId,
191-
LoadbalancerName: *ins.LoadBalancerName,
192-
}
193-
if !util.IsZero(ins.Domain) {
194-
lbInfo.Hostname = ins.Domain
176+
for len(lbIds) > 0 {
177+
lbs := lbIds
178+
if len(lbIds) > 19 { // 分页查询
179+
lbs = lbIds[:19]
180+
lbIds = lbIds[19:]
195181
} else {
196-
vips := util.ConvertPtrSlice(ins.LoadBalancerVips)
197-
if len(vips) > 0 {
198-
lbInfo.Ips = vips
182+
lbIds = nil
183+
}
184+
res, err := ApiCall(context.Background(), false, "DescribeLoadBalancers", region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeLoadBalancersRequest, res *clb.DescribeLoadBalancersResponse, err error) {
185+
req = clb.NewDescribeLoadBalancersRequest()
186+
req.LoadBalancerIds = common.StringPtrs(lbs)
187+
res, err = client.DescribeLoadBalancersWithContext(ctx, req)
188+
return
189+
})
190+
if err != nil {
191+
return nil, errors.WithStack(err)
192+
}
193+
if *res.Response.TotalCount == 0 || len(res.Response.LoadBalancerSet) == 0 {
194+
return nil, nil
195+
}
196+
for _, ins := range res.Response.LoadBalancerSet {
197+
lbInfo := &CLBInfo{
198+
LoadbalancerID: *ins.LoadBalancerId,
199+
LoadbalancerName: *ins.LoadBalancerName,
199200
}
201+
if !util.IsZero(ins.Domain) {
202+
lbInfo.Hostname = ins.Domain
203+
} else {
204+
vips := util.ConvertPtrSlice(ins.LoadBalancerVips)
205+
if len(vips) > 0 {
206+
lbInfo.Ips = vips
207+
}
208+
}
209+
info[*ins.LoadBalancerId] = lbInfo
210+
insIds = append(insIds, ins.LoadBalancerId)
200211
}
201-
info[*ins.LoadBalancerId] = lbInfo
202-
insIds = append(insIds, ins.LoadBalancerId)
203212
}
204213
vpcClient := vpcpkg.GetClient(region)
205-
addrResp, err := ApiCall(context.Background(), "DescribeAddresses", region, func(ctx context.Context, client *clb.Client) (req *vpc.DescribeAddressesRequest, res *vpc.DescribeAddressesResponse, err error) {
206-
req = vpc.NewDescribeAddressesRequest()
207-
req.Filters = []*vpc.Filter{
208-
{
209-
Name: common.StringPtr("instance-id"),
210-
Values: insIds,
211-
},
214+
for len(insIds) > 0 {
215+
ids := insIds
216+
if len(insIds) > 99 { // 分页查询
217+
ids = insIds[:99]
218+
insIds = insIds[99:]
219+
} else {
220+
insIds = nil
212221
}
213-
res, err = vpcClient.DescribeAddressesWithContext(ctx, req)
214-
return
215-
})
216-
if err != nil {
217-
return nil, errors.WithStack(err)
218-
}
219-
for _, addr := range addrResp.Response.AddressSet {
220-
log.FromContext(ctx).V(3).Info("got clb eip addr", "instanceId", addr.InstanceId)
221-
if addr.InstanceId != nil {
222-
if lbInfo, ok := info[*addr.InstanceId]; ok {
223-
log.FromContext(ctx).V(3).Info("set clb eip addr", "instanceId", addr.InstanceId, "eip", *addr.AddressIp)
224-
lbInfo.Ips = []string{*addr.AddressIp}
222+
addrResp, err := ApiCall(context.Background(), false, "DescribeAddresses", region, func(ctx context.Context, client *clb.Client) (req *vpc.DescribeAddressesRequest, res *vpc.DescribeAddressesResponse, err error) {
223+
req = vpc.NewDescribeAddressesRequest()
224+
req.Filters = []*vpc.Filter{
225+
{
226+
Name: common.StringPtr("instance-id"),
227+
Values: ids,
228+
},
229+
}
230+
res, err = vpcClient.DescribeAddressesWithContext(ctx, req)
231+
return
232+
})
233+
if err != nil {
234+
return nil, errors.WithStack(err)
235+
}
236+
for _, addr := range addrResp.Response.AddressSet {
237+
log.FromContext(ctx).V(3).Info("got clb eip addr", "instanceId", addr.InstanceId)
238+
if addr.InstanceId != nil {
239+
if lbInfo, ok := info[*addr.InstanceId]; ok {
240+
log.FromContext(ctx).V(3).Info("set clb eip addr", "instanceId", addr.InstanceId, "eip", *addr.AddressIp)
241+
lbInfo.Ips = []string{*addr.AddressIp}
242+
}
225243
}
226244
}
227245
}

‎pkg/clb/listener.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func GetListenerByIdOrPort(ctx context.Context, region, lbId string, listenerId
7070
}
7171

7272
func GetListenerByPort(ctx context.Context, region, lbId string, port int64, protocol string) (lis *Listener, err error) {
73-
res, err := ApiCall(ctx, "DescribeListeners", region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeListenersRequest, res *clb.DescribeListenersResponse, err error) {
73+
res, err := ApiCall(ctx, false, "DescribeListeners", region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeListenersRequest, res *clb.DescribeListenersResponse, err error) {
7474
req = clb.NewDescribeListenersRequest()
7575
req.Port = &port
7676
req.LoadBalancerId = &lbId
@@ -153,7 +153,7 @@ func CreateListener(ctx context.Context, region, lbId string, port, endPort int6
153153
defer mu.Unlock()
154154
before := time.Now()
155155
resp, err := client.CreateListenerWithContext(ctx, req)
156-
LogAPI(ctx, "CreateListener", req, resp, time.Since(before), err)
156+
LogAPI(ctx, true, "CreateListener", req, resp, time.Since(before), err)
157157
if err != nil {
158158
err = errors.WithStack(err)
159159
return

‎pkg/clb/quota.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func DescribeQuota(ctx context.Context, region string) (quotaMap map[string]int6
123123
req := clb.NewDescribeQuotaRequest()
124124
before := time.Now()
125125
resp, err := client.DescribeQuotaWithContext(ctx, req)
126-
LogAPI(ctx, "DescribeQuota", req, resp, time.Since(before), err)
126+
LogAPI(ctx, false, "DescribeQuota", req, resp, time.Since(before), err)
127127
if err != nil {
128128
return nil, errors.WithStack(err)
129129
}

‎pkg/clb/target.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ func ContainsTarget(ctx context.Context, region, lbId string, port int64, protoc
2525
client := GetClient(region)
2626
before := time.Now()
2727
resp, err := client.DescribeTargets(req)
28-
LogAPI(ctx, "DescribeTargets", req, resp, time.Since(before), err)
28+
LogAPI(ctx, false, "DescribeTargets", req, resp, time.Since(before), err)
2929
if err != nil {
3030
return false, err
3131
}
@@ -176,7 +176,7 @@ func DeregisterTargetsForListener(ctx context.Context, region, lbId, listenerId
176176
client := GetClient(region)
177177
before := time.Now()
178178
resp, err := client.DeregisterTargetsWithContext(ctx, req)
179-
LogAPI(ctx, "DeregisterTargets", req, resp, time.Since(before), err)
179+
LogAPI(ctx, true, "DeregisterTargets", req, resp, time.Since(before), err)
180180
if err != nil {
181181
return err
182182
}
@@ -206,7 +206,7 @@ func RegisterTargets(ctx context.Context, region, lbId, listenerId string, targe
206206
defer mu.Unlock()
207207
before := time.Now()
208208
resp, err := client.RegisterTargetsWithContext(ctx, req)
209-
LogAPI(ctx, "RegisterTargets", req, resp, time.Since(before), err)
209+
LogAPI(ctx, true, "RegisterTargets", req, resp, time.Since(before), err)
210210
if err != nil {
211211
return err
212212
}
@@ -254,7 +254,7 @@ func DescribeTargets(ctx context.Context, region, lbId, listenerId string) (targ
254254
client := GetClient(region)
255255
before := time.Now()
256256
resp, err := client.DescribeTargetsWithContext(ctx, req)
257-
LogAPI(ctx, "DescribeTargets", req, resp, time.Since(before), err)
257+
LogAPI(ctx, false, "DescribeTargets", req, resp, time.Since(before), err)
258258
if err != nil {
259259
return
260260
}

‎pkg/clb/wait.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ func Wait(ctx context.Context, region, reqId, taskName string) (ids []string, er
1818
err = ctx.Err()
1919
return
2020
default:
21-
res, err := ApiCall(context.Background(), "DescribeTaskStatus", region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeTaskStatusRequest, res *clb.DescribeTaskStatusResponse, err error) {
21+
res, err := ApiCall(context.Background(), false, "DescribeTaskStatus", region, func(ctx context.Context, client *clb.Client) (req *clb.DescribeTaskStatusRequest, res *clb.DescribeTaskStatusResponse, err error) {
2222
req = clb.NewDescribeTaskStatusRequest()
2323
req.TaskId = &reqId
2424
res, err = client.DescribeTaskStatusWithContext(ctx, req)

‎pkg/vpc/client.go‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,11 @@
11
package vpc
22

33
import (
4-
"context"
5-
6-
"github.com/go-logr/logr"
74
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
85
vpc "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312"
96
"github.com/tkestack/tke-extend-network-controller/pkg/cloudapi"
107
"github.com/tkestack/tke-extend-network-controller/pkg/clusterinfo"
118
ctrl "sigs.k8s.io/controller-runtime"
12-
"sigs.k8s.io/controller-runtime/pkg/log"
139
)
1410

1511
var vpcLog = ctrl.Log.WithName("vpc")
@@ -30,13 +26,3 @@ func GetClient(region string) *vpc.Client {
3026
clients[region] = client
3127
return client
3228
}
33-
34-
func LogAPI(ctx context.Context, apiName string, req, resp any, err error) {
35-
var logger logr.Logger
36-
if ctx != nil {
37-
logger = log.FromContext(ctx)
38-
} else {
39-
logger = vpcLog
40-
}
41-
logger.Info("VPC API Call", "api", apiName, "request", req, "response", resp, "error", err)
42-
}

0 commit comments

Comments
 (0)