Skip to content

Commit 191a005

Browse files
luyiismeluyiisme
authored
enable linux os metrics (#38)
- 增加 linux os 的metrics的采集和上报； - 优化主动上报时逻辑，优化连接使用； - 其他修复若干；
1 parent 87c4605 commit 191a005

File tree

44 files changed

+841
-433
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+841
-433
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
[English Document](./README_EN.md)
99

10-
访问 [WIKI](https://github.com/alipay/sofa-lookout/wiki) 查看完整的文档使用指南。
10+
访问 [WIKI](http://www.sofastack.tech/sofa-lookout/docs/Home) 查看完整的文档使用指南。
1111

1212
SOFALookout 是一个利用多维度的 metrics 对目标系统进行度量和监控的项目。SOFALookout 的多维度 metrics 参考[Metrics2.0 标准](http://metrics20.org/)。SOFALookout 项目分为客户端部分与服务器端部分。
1313

‎client/README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ SOFALookout Client 项目支持 Maven 3.2.5+，JDK 6+ 进行编译。
1010

1111
## 2. API 埋点需知
1212

13-
lookout-api 支持被单独依赖和使用，方便植入您的项目代码，收集需要的 metrics，更多信息参考 [WIKI 文档](https://github.com/alipay/sofa-lookout/wiki)。
13+
lookout-api 支持被单独依赖和使用，方便植入您的项目代码，收集需要的 metrics，更多信息参考 [WIKI 文档](http://www.sofastack.tech/sofa-lookout/docs/Home)。
1414

1515
## 3. 扩展能力
1616

@@ -49,4 +49,4 @@ lookout 客户端提供了 SPI 机制（只需要实现 `com.alipay.lookout.spi.
4949

5050
## 6.如何使用
5151

52-
参考 [WIKI 文档](https://github.com/alipay/sofa-lookout/wiki)的快速开始和用户手册。
52+
参考 [WIKI 文档](http://www.sofastack.tech/sofa-lookout/docs/Home)的快速开始和用户手册。

‎client/lookout-client/src/main/java/com/alipay/lookout/client/AbstractLookoutClient.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ protected void addDefaultCommonTags(CommonTagsAccessor commonTagsAccessor) {
9191
if (StringUtils.isNotEmpty(zone)) {
9292
commonTagsAccessor.setCommonTag("zone", zone);
9393
}
94-
//Ant cloud middleware instanceId
94+
//instanceId
9595
String instanceId = System.getProperty(INSTANCE_ID_NAME);
9696
if (StringUtils.isNotEmpty(instanceId)) {
9797
commonTagsAccessor.setCommonTag("instance_id", instanceId);

‎client/lookout-client/src/main/java/com/alipay/lookout/client/DefaultLookoutClient.java

+8
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@
1818

1919
import com.alipay.lookout.api.Lookout;
2020
import com.alipay.lookout.api.MetricRegistry;
21+
import com.alipay.lookout.core.config.MetricConfig;
2122
import com.alipay.lookout.remote.step.LookoutRegistry;
2223

24+
import static com.alipay.lookout.core.config.LookoutConfig.LOOKOUT_EXPORTER_ENABLE;
25+
2326
/**
2427
* Created by [email protected] on 2017/10/4.
2528
*/
@@ -44,7 +47,12 @@ public synchronized void addRegistry(MetricRegistry registry) {
4447
registry.registerExtendedMetrics();
4548
}
4649
super.addRegistry(registry);
50+
4751
if (registry instanceof LookoutRegistry) {
52+
MetricConfig config = ((LookoutRegistry) registry).getConfig();
53+
if (!config.getBoolean(LOOKOUT_EXPORTER_ENABLE, false)) {
54+
return;
55+
}
4856
try {
4957
setMetricsHttpExporter(PollerUtils.exportHttp((LookoutRegistry) registry));
5058
} catch (Exception e) {

‎client/lookout-client/src/main/java/com/alipay/lookout/client/SimpleLookoutClient.java

+5
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929

3030
import java.util.concurrent.atomic.AtomicInteger;
3131

32+
import static com.alipay.lookout.core.config.LookoutConfig.LOOKOUT_EXPORTER_ENABLE;
33+
3234
/**
3335
* 与 @see DefaultLookoutClient 相比，SimpleLookoutClient限制更严格：
3436
* 1.所有的Registry需要预先设置！
@@ -78,6 +80,9 @@ public SimpleLookoutClient(String appName, LookoutConfig config, MetricRegistry.
7880
registry.registerExtendedMetrics();
7981
super.addRegistry(registry);
8082
if (registry instanceof LookoutRegistry) {
83+
if (!lookoutConfig.getBoolean(LOOKOUT_EXPORTER_ENABLE, false)) {
84+
return;
85+
}
8186
try {
8287
setMetricsHttpExporter(PollerUtils.exportHttp((LookoutRegistry) registry));
8388
} catch (Exception e) {

‎client/lookout-common/src/main/java/com/alipay/lookout/common/utils/PriorityTagUtil.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public static PRIORITY resolve(Iterable<Tag> tags) {
3636
String value = Utils.getTagValue(tags, TAG_PRIORITY_KEY);
3737
if (value != null) {
3838
for (PRIORITY p : PRIORITY.values()) {
39-
if (p.name().equals(value)) {
39+
if (p.name().equalsIgnoreCase(value)) {
4040
return p;
4141
}
4242
}

‎client/lookout-common/src/main/resources/com/alipay/lookout/log/log4j2/log-conf.xml

-16
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,6 @@
3535
</Policies>
3636
</RollingFile>
3737

38-
<RollingFile name="ANTEVENTCONTEXT-APPENDER" fileName="${LOOKOUT_LOG_PATH}/lookout/ant-biz-stat.log" append="true"
39-
filePattern="${LOOKOUT_LOG_PATH}/lookout/ant-biz-stat-%d{yyyy-MM-dd}.log">
40-
<ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
41-
<PatternLayout charset="${LOOKOUT_FILE_ENCODING}">
42-
<pattern>%d %-5p %-32t - %m%n</pattern>
43-
</PatternLayout>
44-
<Policies>
45-
<!-- 按天分日志文件:重要的是 filePattern 配置到按照天 -->
46-
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
47-
</Policies>
48-
</RollingFile>
4938

5039
</appenders>
5140

@@ -54,10 +43,5 @@
5443
<appender-ref ref="ROOT-APPENDER"/>
5544
<appender-ref ref="ERROR-APPENDER"/>
5645
</root>
57-
58-
<logger name="ANTEVENTCONTEXT-LOG" value="INFO" additivity="false">
59-
<appender-ref ref="ANTEVENTCONTEXT-APPENDER"/>
60-
<appender-ref ref="ERROR-APPENDER" />
61-
</logger>
6246
</loggers>
6347
</configuration>

‎client/lookout-common/src/test/java/com/alipay/lookout/common/utils/PriorityTagUtilTest.java

+9
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.alipay.lookout.api.BasicTag;
2020
import com.alipay.lookout.api.PRIORITY;
2121
import com.alipay.lookout.api.Tag;
22+
import com.alipay.lookout.common.LookoutConstants;
2223
import org.junit.Assert;
2324
import org.junit.Test;
2425

@@ -40,6 +41,14 @@ public void testResolveHigh() {
4041
Assert.assertEquals(PRIORITY.HIGH, PriorityTagUtil.resolve(tags));
4142
}
4243

44+
@Test
45+
public void testResolveHigh2() {
46+
List<Tag> tags = new ArrayList<Tag>();
47+
tags.add(new BasicTag("k1", "v1"));
48+
tags.add(LookoutConstants.HIGH_PRIORITY_TAG);
49+
Assert.assertEquals(PRIORITY.HIGH, PriorityTagUtil.resolve(tags));
50+
}
51+
4352
@Test
4453
public void testResolveNothing() {
4554
List<Tag> tags = new ArrayList<Tag>();

‎client/lookout-core/src/main/java/com/alipay/lookout/core/DefaultDistributionSummary.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public void record(long amount) {
5454
public Indicator measure() {
5555
long now = clock.wallTime();
5656
return new Indicator(now, id).addMeasurement(Statistic.count.name(), count.get())
57-
.addMeasurement(Statistic.count.name(), totalAmount.get());
57+
.addMeasurement(Statistic.totalAmount.name(), totalAmount.get());
5858
}
5959

6060
@Override

‎client/lookout-core/src/main/java/com/alipay/lookout/core/config/LookoutConfig.java

+6-5
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,17 @@ public final class LookoutConfig extends MapConfiguration implements MetricConfi
4040
public static final String LOOKOUT_AUTOPOLL_OS_METRIC_IGNORE = "lookout.autopoll.os.ignore";
4141
public static final String LOOKOUT_AGENT_SERVER_PORT = "lookout.agent.server.port";
4242
public static final String LOOKOUT_ANT_EVENT_LOG_ENABLE = "lookout.ant.event.log.enable";
43+
/**
44+
* 多久没有请求拉取数据就进入idle状态
45+
*/
46+
public static final String LOOKOUT_EXPORTER_IDLE_SECONDS = "lookout.exporter.idle.seconds";
47+
public static final String LOOKOUT_EXPORTER_ENABLE = "lookout.exporter.enable";
4348
public static final String LOOKOUT_EXPORTER_ACCESS_TOKEN = "lookout.exporter.access.token";
4449

4550
public static final String LOOKOUT_PROMETHEUS_EXPORTER_SERVER_PORT = "lookout.prometheus.exporter.server.port";
4651
// default value
4752
public static final int DEFAULT_WEB_SERVER_PORT = 8083;
53+
public static final int DEFAULT_HTTP_EXPORTER_PORT = 19399;
4854
public static final int DEFAULT_PROMETHEUS_EXPORTER_SERVER_PORT = 9494;
4955
public static final String APP_NAME = "app.name";
5056
public static int DEFAULT_REPORT_BATCH_SIZE = 1700;
@@ -55,11 +61,6 @@ public final class LookoutConfig extends MapConfiguration implements MetricConfi
5561
*/
5662
// public static final String POLLER_EXPORTER_ENABLED = "lookout.poller.enabled";
5763

58-
/**
59-
* 多久没有请求拉取数据就进入idle状态
60-
*/
61-
public static final String POLLER_EXPORTER_IDLE_SECONDS = "lookout.poller.exporter.idle.seconds";
62-
6364
/**
6465
* priority->Millsecond
6566
**/

‎client/lookout-ext-os/src/main/java/com/alipay/lookout/os/linux/DiskUsageMetricsImporter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public DiskUsageMetricsImporter(String filePath, long timeout, TimeUnit timeoutU
8383
protected void doRegister(Registry registry) {
8484
for (Map.Entry<String, DiskUsage> entry : diskUsageByDevice.entrySet()) {
8585
final String device = entry.getKey();
86-
Id id = registry.createId("os.disk.usage." + device);
86+
Id id = registry.createId("os.disk.usage");
8787
id = id.withTag("device", device);
8888
id = id.withTag("root", diskUsageByDevice.get(device).fsFile);
8989
id = id.withTag("type", diskUsageByDevice.get(device).fsVfType);

‎client/lookout-ext-os/src/main/java/com/alipay/lookout/os/linux/IOStatsMetricsImporter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public IOStatsMetricsImporter(String filePath, String upTimeFilePath, long timeo
9696
protected void doRegister(Registry registry) {
9797
for (Map.Entry<String, Float[]> entry : statsByDevice.entrySet()) {
9898
final String device = entry.getKey();
99-
Id id = registry.createId("os.io.stats." + device);
99+
Id id = registry.createId("os.io.stats");
100100
id = id.withTag("device", device);
101101
MixinMetric mixin = registry.mixinMetric(id);
102102

‎client/lookout-ext-os/src/main/java/com/alipay/lookout/os/linux/NetTrafficMetricsImporter.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,10 @@ public class NetTrafficMetricsImporter extends CachedMetricsImporter {
7777

7878
private static final Pattern NET_PATTERN = Pattern.compile(STR_PATTERN);
7979

80-
private static final String[] FIELDS = { "net.in.bytes", "net.in.packets",
81-
"net.in.errs", "net.in.dropped", "net.in.fifo.errs", "net.in.frame.errs",
82-
"net.in.compressed", "net.in.multicast", "net.out.bytes", "net.out.packets",
83-
"net.out.errs", "net.out.dropped", "net.out.fifo.errs", "net.out.collisions",
84-
"net.out.carrier.errs", "net.out.compressed" };
80+
private static final String[] FIELDS = { "in.bytes", "in.packets", "in.errs",
81+
"in.dropped", "in.fifo.errs", "in.frame.errs", "in.compressed", "in.multicast",
82+
"out.bytes", "out.packets", "out.errs", "out.dropped", "out.fifo.errs",
83+
"out.collisions", "out.carrier.errs", "out.compressed" };
8584

8685
private String filePath;
8786
private Map<String, Long[]> statByFace;
@@ -104,7 +103,7 @@ public NetTrafficMetricsImporter(String filePath, long timeout, TimeUnit timeout
104103
protected void doRegister(Registry registry) {
105104
for (final Map.Entry<String, Long[]> entry : statByFace.entrySet()) {
106105
final String face = entry.getKey();
107-
Id id = registry.createId("os.net.stat." + face);
106+
Id id = registry.createId("os.net.stats").withTag("intfc", face);
108107
MixinMetric mixin = registry.mixinMetric(id);
109108

110109
for (int i = 0; i < entry.getValue().length; i++) {

‎client/lookout-reg-prometheus/src/test/java/com/alipay/lookout/reg/prometheus/ExporterServerTest.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ public class ExporterServerTest {
3333

3434
@Test
3535
public void testExporterServer() throws IOException {
36-
ExporterServer server = new ExporterServer(9494);
36+
ExporterServer server = new ExporterServer(9194);
3737
server.start();
38-
String result = sendHttpRequest(new URL("http://localhost:9494/"));
38+
String result = sendHttpRequest(new URL("http://localhost:9194/"));
3939
Assert.assertTrue(result.contains("/metrics"));
4040
server.stop();
4141
}

‎client/lookout-reg-server/src/main/java/com/alipay/lookout/remote/report/AddressService.java

+14
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,19 @@
1616
*/
1717
package com.alipay.lookout.remote.report;
1818

19+
import java.util.Random;
20+
1921
/**
2022
* 动态地址服务
2123
* Created by [email protected] on 2017/3/8.
2224
*/
2325
public interface AddressService {
26+
ThreadLocal<Random> randomThreadLocal = new ThreadLocal<Random>() {
27+
@Override
28+
protected Random initialValue() {
29+
return new Random();
30+
}
31+
};
2432

2533
/**
2634
* set a agentTestUrl,优先与动态发现的地址
@@ -38,8 +46,14 @@ public interface AddressService {
3846

3947
boolean isAgentServerExisted();
4048

49+
/**
50+
* select one address from addresses
51+
*
52+
* @return agent address
53+
*/
4154
Address getAgentServerHost();
4255

56+
@Deprecated
4357
void clearAddressCache();
4458

4559
}

‎client/lookout-reg-server/src/main/java/com/alipay/lookout/remote/report/DefaultAddressService.java

+45-8
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,21 @@
1616
*/
1717
package com.alipay.lookout.remote.report;
1818

19+
import com.google.common.base.Splitter;
1920
import com.google.common.base.Strings;
21+
import com.google.common.collect.Lists;
22+
23+
import java.util.ArrayList;
24+
import java.util.List;
2025

2126
/**
2227
* 默认静态地址服务
2328
* with vip and test model url
2429
* Created by [email protected] on 2017/5/31.
2530
*/
2631
public class DefaultAddressService implements AddressService {
27-
28-
private Address agentServerVip;
29-
private Address agentTestUrl;
32+
private Address agentTestUrl;
33+
private List<Address> addressList;
3034

3135
public DefaultAddressService() {
3236
}
@@ -35,7 +39,6 @@ public DefaultAddressService(String appName) {
3539
}
3640

3741
// cache,for a connection keep alive & reuse;
38-
3942
public void clearAddressCache() {
4043
}
4144

@@ -46,21 +49,55 @@ public void setAgentTestUrl(String agentTestUrl) {
4649
}
4750

4851
public void setAgentServerVip(String agentServerVip) {
49-
if (!Strings.isNullOrEmpty(agentServerVip)) {
50-
this.agentServerVip = new Address(agentServerVip);
52+
if (Strings.isNullOrEmpty(agentServerVip)) {
53+
return;
54+
}
55+
//multi addresses
56+
if (agentServerVip.contains(",")) {
57+
List<String> agentServers = Splitter.on(',').splitToList(agentServerVip);
58+
setAddressList(agentServers);
59+
return;
60+
}
61+
this.addressList = Lists.newArrayList(new Address(agentServerVip));
62+
}
63+
64+
public void setAddressList(List<String> addresses) {
65+
if (addresses == null || addresses.isEmpty()) {
66+
return;
5167
}
68+
List<Address> addressList = new ArrayList<Address>();
69+
for (String addressStr : addresses) {
70+
addressList.add(new Address(addressStr.trim()));
71+
}
72+
this.addressList = addressList;
73+
}
74+
75+
protected Address getAgentTestUrl() {
76+
return agentTestUrl;
77+
}
78+
79+
protected List<Address> getAddressList() {
80+
return addressList;
5281
}
5382

5483
@Override
5584
public boolean isAgentServerExisted() {
56-
return agentTestUrl != null || agentServerVip != null;
85+
return agentTestUrl != null || (addressList != null && !addressList.isEmpty());
5786
}
5887

5988
@Override
6089
public Address getAgentServerHost() {
6190
if (agentTestUrl != null) {
6291
return agentTestUrl;
6392
}
64-
return agentServerVip;
93+
List<Address> addrList = addressList;
94+
if (addrList == null) {
95+
return null;
96+
}
97+
if (addrList.size() == 1) {
98+
return addrList.get(0);
99+
}
100+
int randomNodeIdx = randomThreadLocal.get().nextInt(addrList.size());
101+
return addrList.get(randomNodeIdx);
65102
}
66103
}

0 commit comments

Comments
 (0)