Skip to content

Commit bf70e43

Browse files
luyiismeluyiisme
authored
fix TopUtil concurrency problems and http reporter support dynamic filtering configurations
fix TopUtili bug (#54) Http reporter support dynamic filtering configurations #55
1 parent e8cba52 commit bf70e43

File tree

35 files changed

+543
-56
lines changed

35 files changed

+543
-56
lines changed

‎README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@ SOFALookout 是一个利用多维度的 metrics 对目标系统进行度量和
3232
- 服务器端代码编译，进入server子目录，Maven 3.2.5+, JDK Version 1.8+；
3333
## 样例工程
3434

35-
样例工程演示了如何快速使用 SOFALookout，[详细可参考](https://github.com/alipay/sofa-lookout/wiki/useguide-samples)。
35+
样例工程演示了如何快速使用 SOFALookout，[详细可参考](https://github.com/sofastack/sofa-lookout/wiki/useguide-samples)。
3636

3737
## 贡献
3838
如何参与 SOFALookout [代码贡献](./CONTRIBUTING.md)
3939

4040
## 开源许可
41-
SOFALookout 基于 [Apache License 2.0](./LICENSE) 协议，SOFALookout 依赖了一些三方组件，它们的开源协议参见[依赖组件版权说明](https://github.com/alipay/sofa-lookout/wiki/NOTICE)。
41+
SOFALookout 基于 [Apache License 2.0](./LICENSE) 协议，SOFALookout 依赖了一些三方组件，它们的开源协议参见[依赖组件版权说明](https://github.com/sofastack/sofa-lookout/wiki/NOTICE)。

‎README_EN.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
![license](https://img.shields.io/badge/license-Apache--2.0-green.svg)
66
![maven](https://img.shields.io/github/release/alipay/sofa-lookout.svg)
77

8-
Visit [WIKI](https://github.com/alipay/sofa-lookout/wiki) for the full documentation, examples, and guides.
8+
Visit [WIKI](https://github.com/sofastack/sofa-lookout/wiki) for the full documentation, examples, and guides.
99

1010
Lookout can help you to measure and monitor the status of the target system with its multi-dimensional metrics.
1111
Lookout's multi-dimensional metrics refer to the [Metrics 2.0](http://metrics20.org/) standard.

‎client/lookout-api/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<groupId>com.alipay.sofa.lookout</groupId>
77
<artifactId>lookout-client-parent</artifactId>
8-
<version>1.5.3</version>
8+
<version>1.5.4</version>
99
<relativePath>../pom.xml</relativePath>
1010
</parent>
1111

‎client/lookout-api/src/main/java/com/alipay/lookout/common/top/AbstractTopMetric.java

+25-19
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121

2222
import java.util.Comparator;
2323
import java.util.TreeSet;
24+
import java.util.concurrent.locks.Lock;
25+
import java.util.concurrent.locks.ReentrantLock;
2426

2527
/**
2628
* Created by [email protected] on 2017/3/31.
@@ -44,6 +46,7 @@ public int compare(TopUtil.Entry<Id, Long> o1,
4446
};
4547
private final TopUtil.Order order;
4648
private long lastRolledStamp = -1l;
49+
private final Lock lock = new ReentrantLock();
4750

4851
AbstractTopMetric(Registry registry, Id id, int maxNumber, TopUtil.Order order) {
4952
this.maxNumber = maxNumber;
@@ -57,32 +60,35 @@ protected void pushAsync(Long value, Id timerId) {
5760
TopUtil.executor.execute(new Runnable() {
5861
@Override
5962
public void run() {
60-
pushSafe(set, entry);
63+
lock.lock();
64+
try {
65+
pushSafe(set, entry);
66+
} finally {
67+
lock.unlock();
68+
}
6169
}
6270
});
6371
}
6472

6573
private void pushSafe(TreeSet<TopUtil.Entry<Id, Long>> set, TopUtil.Entry<Id, Long> e) {
66-
synchronized (set) {
67-
if (!set.isEmpty()) {
68-
boolean replaceable = false;
69-
TopUtil.Entry<Id, Long> boundaryTarget = null;
70-
if (order == TopUtil.Order.DESC) {
71-
boundaryTarget = set.first();
72-
replaceable = boundaryTarget.getValue() < e.getValue();
73-
} else {
74-
boundaryTarget = set.last();
75-
replaceable = boundaryTarget.getValue() > e.getValue();
76-
}
77-
if (replaceable & set.size() >= maxNumber) {
78-
remove(set, boundaryTarget);
79-
}
80-
if (!replaceable & set.size() >= maxNumber) {
81-
return;//不add
82-
}
74+
if (!set.isEmpty()) {
75+
boolean replaceable = false;
76+
TopUtil.Entry<Id, Long> boundaryTarget = null;
77+
if (order == TopUtil.Order.DESC) {
78+
boundaryTarget = set.first();
79+
replaceable = boundaryTarget.getValue() < e.getValue();
80+
} else {
81+
boundaryTarget = set.last();
82+
replaceable = boundaryTarget.getValue() > e.getValue();
83+
}
84+
if (replaceable & set.size() >= maxNumber) {
85+
remove(set, boundaryTarget);
86+
}
87+
if (!replaceable & set.size() >= maxNumber) {
88+
return;//不add
8389
}
84-
add(set, e);
8590
}
91+
add(set, e);
8692
}
8793

8894
private void remove(TreeSet<TopUtil.Entry<Id, Long>> set, TopUtil.Entry<Id, Long> boundaryTarget) {

‎client/lookout-api/src/main/java/com/alipay/lookout/common/top/TopUtil.java

+3-5
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,11 @@ public static TopGauger topGauger(final Registry registry, final Id id, final in
7373
if (registry instanceof NoopRegistry || (id == NoopRegistry.INSTANCE.createId(null))) {
7474
return NoopTopGauger.INSTANCE;
7575
}
76-
Id key = id.withTag(TOP_NUM_TAG_KEY, String.valueOf(maxNumber));
76+
final Id key = id.withTag(TOP_NUM_TAG_KEY, String.valueOf(maxNumber));
7777
TopGauger topGauger = computeIfAbsent(cache, key, new Function<Object, TopGauger>() {
7878
@Override
7979
public TopGauger apply(Object obj) {
80-
return new DefaultTopGauger(registry, id, maxNumber, order);
80+
return new DefaultTopGauger(registry, key, maxNumber, order);
8181
}
8282
});
8383
return topGauger == null ? NoopTopGauger.INSTANCE : topGauger;
@@ -109,9 +109,7 @@ public enum Order {
109109
* @param <K>
110110
* @param <V>
111111
*/
112-
static class Entry<K, V>
113-
114-
{
112+
static class Entry<K, V> {
115113
private final K key;
116114
private V value;
117115

‎client/lookout-client/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-common/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-common/src/test/java/com/alipay/lookout/common/utils/ClassUtilTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,9 @@ public void testNewInstance() {
3535
new Object[] { 5 });
3636
Assert.assertEquals("5", integer.toString());
3737
}
38+
39+
@Test(expected = IllegalStateException.class)
40+
public void testNewInstanceException() {
41+
ClassUtil.newInstance(ArrayList.class.getName() + "x", null, null);
42+
}
3843
}

‎client/lookout-core/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-ext-jvm/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-ext-os/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<groupId>com.alipay.sofa.lookout</groupId>
77
<artifactId>lookout-client-parent</artifactId>
8-
<version>1.5.3</version>
8+
<version>1.5.4</version>
99
<relativePath>../pom.xml</relativePath>
1010
</parent>
1111

‎client/lookout-reg-dropwizard/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-reg-prometheus/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-reg-server/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<parent>
77
<groupId>com.alipay.sofa.lookout</groupId>
88
<artifactId>lookout-client-parent</artifactId>
9-
<version>1.5.3</version>
9+
<version>1.5.4</version>
1010
<relativePath>../pom.xml</relativePath>
1111
</parent>
1212

‎client/lookout-reg-server/src/main/java/com/alipay/lookout/remote/report/HttpObserver.java

+18-6
Original file line numberDiff line numberDiff line change
@@ -158,14 +158,27 @@ public void update(List<LookoutMeasurement> measures, Map<String, String> metada
158158
logger.debug("## no gateway address found, drop metrics:\n{}\n", measures.toString());
159159
return;
160160
}
161-
logger.debug(">> send metrics to {}:\n{}\n", address, measures.toString());
162-
List<List<LookoutMeasurement>> batches = getBatches(measures,
161+
162+
//Filter measures by config
163+
List<LookoutMeasurement> filteredMeasures = filter(measures);
164+
if (filteredMeasures.isEmpty()) {
165+
return;
166+
}
167+
logger.debug(">> send metrics to {}:\n{}\n", address, filteredMeasures);
168+
List<List<LookoutMeasurement>> batches = getBatches(filteredMeasures,
163169
lookoutConfig.getInt(LOOKOUT_REPORT_BATCH_SIZE, DEFAULT_REPORT_BATCH_SIZE));
164170
for (List<LookoutMeasurement> batch : batches) {
165171
reportBatch(batch, metadata, address);
166172
}
167173
}
168174

175+
private List<LookoutMeasurement> filter(List<LookoutMeasurement> measures) {
176+
if (httpRequestProcessor instanceof ReportDecider) {
177+
return ((ReportDecider) httpRequestProcessor).filter(measures);
178+
}
179+
return measures;
180+
}
181+
169182
/**
170183
* Get a list of all measurements and break them into batches.
171184
*
@@ -255,17 +268,16 @@ void sendHttpDataSilently(HttpRequest httpRequest, Map<String, String> metadata)
255268
} catch (Throwable e) {
256269
if (e instanceof UnknownHostException || e instanceof ConnectException) {
257270
addressService.clearAddressCache();
258-
logger.info(">>WARNING: lookout agent:{} err?cause:{}", httpRequest.toString(),
259-
e.getMessage());
271+
logger
272+
.info(">>WARNING: lookout agent:{} err?cause:{}", httpRequest, e.getMessage());
260273
} else if (e instanceof SocketTimeoutException) {
261274
registry().counter(
262275
registry().createId("lookout.client.report.fail.count").withTag("err",
263276
"socket_timeout")).inc();
264277
} else {
265278
registry().counter(registry().createId("lookout.client.report.fail.count")).inc();
266279
}
267-
logger.info(">>WARNING: lookout agent:{} fail!cause:{}", httpRequest.toString(),
268-
e.getMessage());
280+
logger.info(">>WARNING: lookout agent:{} fail!cause:{}", httpRequest, e.getMessage());
269281
}
270282
}
271283

‎client/lookout-reg-server/src/main/java/com/alipay/lookout/remote/report/poller/MetricsHttpExporter.java

+7
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import com.alibaba.fastjson.JSON;
2020
import com.alibaba.fastjson.JSONObject;
21+
import com.alipay.lookout.common.log.LookoutLoggerFactory;
2122
import com.alipay.lookout.common.utils.CommonUtil;
2223
import com.google.common.collect.Sets;
2324
import com.sun.net.httpserver.HttpExchange;
@@ -26,6 +27,7 @@
2627
import org.apache.commons.lang3.StringUtils;
2728
import org.apache.http.NameValuePair;
2829
import org.apache.http.client.utils.URIBuilder;
30+
import org.slf4j.Logger;
2931

3032
import java.io.IOException;
3133
import java.io.OutputStream;
@@ -48,6 +50,8 @@
4850
* @since 2018/7/17
4951
*/
5052
public class MetricsHttpExporter {
53+
static final Logger logger = LookoutLoggerFactory
54+
.getLogger(MetricsHttpExporter.class);
5155
private static final Charset UTF8 = Charset.forName("UTF-8");
5256
private static final int DEFAULT_BACKLOG = 2;
5357
private final PollerController controller;
@@ -146,6 +150,9 @@ public void handle(HttpExchange exchange)
146150

147151
// if (oldRate != newStep || oldSlotCount != newSlotCount) {
148152
// }
153+
} catch (Throwable e) {
154+
logger.warn("pull metrics failed."
155+
+ e.getMessage());
149156
} finally {
150157
exchange.close();
151158
}

0 commit comments

Comments
 (0)