[collector,manager]支持sitemap全站类型监控
This commit is contained in:
tomsun28
@@ -22,6 +22,7 @@ import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
import org.apache.http.client.methods.RequestBuilder;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
@@ -31,13 +32,21 @@ import org.apache.http.protocol.HttpContext;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
import javax.net.ssl.SSLException;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.net.ConnectException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -100,6 +109,8 @@ public class HttpCollectImpl extends AbstractCollect {
|
||||
parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
|
||||
} else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){
|
||||
parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime);
|
||||
} else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) {
|
||||
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
|
||||
} else {
|
||||
parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
|
||||
}
|
||||
@@ -172,6 +183,99 @@ public class HttpCollectImpl extends AbstractCollect {
|
||||
builder.addValues(valueRowBuilder.build());
|
||||
}
|
||||
|
||||
private void parseResponseBySiteMap(String resp, List<String> aliasFields,
|
||||
CollectRep.MetricsData.Builder builder) {
|
||||
List<String> siteUrls = new LinkedList<>();
|
||||
// 使用xml解析
|
||||
boolean isXmlFormat = true;
|
||||
try {
|
||||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||
Document document = db.parse(new ByteArrayInputStream(resp.getBytes(StandardCharsets.UTF_8)));
|
||||
NodeList urlList = document.getElementsByTagName("url");
|
||||
for (int i = 0; i < urlList.getLength(); i++) {
|
||||
Node urlNode = urlList.item(i);
|
||||
NodeList childNodes = urlNode.getChildNodes();
|
||||
for (int k = 0; k < childNodes.getLength(); k++) {
|
||||
Node currentNode = childNodes.item(k);
|
||||
// 区分出text类型的node以及element类型的node
|
||||
if (currentNode.getNodeType() == Node.ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
|
||||
//获取了loc节点的值
|
||||
siteUrls.add(currentNode.getFirstChild().getNodeValue());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(e.getMessage(), e);
|
||||
isXmlFormat = false;
|
||||
}
|
||||
// 若xml解析失败 用txt格式解析
|
||||
if (!isXmlFormat) {
|
||||
try {
|
||||
String[] urls = resp.split("\n");
|
||||
// 校验是否是URL
|
||||
if (IpDomainUtil.isHasSchema(urls[0])) {
|
||||
siteUrls.addAll(Arrays.asList(urls));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
// todo siteUrl 限制数量
|
||||
// 开始循环访问每个site url 采集其 http status code, responseTime, 异常信息
|
||||
int maxUrlNum = 100;
|
||||
for (String siteUrl : siteUrls) {
|
||||
maxUrlNum --;
|
||||
if (maxUrlNum <= 0) {
|
||||
break;
|
||||
}
|
||||
String errorMsg = "";
|
||||
Integer statusCode = null;
|
||||
long startTime = System.currentTimeMillis();
|
||||
try {
|
||||
HttpGet httpGet = new HttpGet(siteUrl);
|
||||
CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet);
|
||||
statusCode = response.getStatusLine().getStatusCode();
|
||||
EntityUtils.consume(response.getEntity());
|
||||
} catch (ClientProtocolException e1) {
|
||||
if (e1.getCause() != null) {
|
||||
errorMsg = e1.getCause().getMessage();
|
||||
} else {
|
||||
errorMsg = e1.getMessage();
|
||||
}
|
||||
} catch (UnknownHostException e2) {
|
||||
// 对端不可达
|
||||
errorMsg = "unknown host";
|
||||
} catch (InterruptedIOException | ConnectException | SSLException e3) {
|
||||
// 对端连接失败
|
||||
errorMsg = "connect error: " + e3.getMessage();
|
||||
} catch (IOException e4) {
|
||||
// 其它IO异常
|
||||
errorMsg = "io error: " + e4.getMessage();
|
||||
} catch (Exception e) {
|
||||
errorMsg = "error: " + e.getMessage();
|
||||
}
|
||||
long responseTime = System.currentTimeMillis() - startTime;
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
for (String alias : aliasFields) {
|
||||
if (CollectorConstants.URL.equalsIgnoreCase(alias)) {
|
||||
valueRowBuilder.addColumns(siteUrl);
|
||||
} else if (CollectorConstants.STATUS_CODE.equalsIgnoreCase(alias)) {
|
||||
valueRowBuilder.addColumns(statusCode == null ?
|
||||
CommonConstants.NULL_VALUE : String.valueOf(statusCode));
|
||||
} else if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
|
||||
valueRowBuilder.addColumns(String.valueOf(responseTime));
|
||||
} else if (CollectorConstants.ERROR_MSG.equalsIgnoreCase(alias)) {
|
||||
valueRowBuilder.addColumns(errorMsg);
|
||||
} else {
|
||||
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
|
||||
}
|
||||
}
|
||||
builder.addValues(valueRowBuilder.build());
|
||||
}
|
||||
}
|
||||
|
||||
private void parseResponseByXmlPath(String resp, List<String> aliasFields, HttpProtocol http,
|
||||
CollectRep.MetricsData.Builder builder) {
|
||||
}
|
||||
@@ -298,7 +402,6 @@ public class HttpCollectImpl extends AbstractCollect {
|
||||
log.error("not support the http method: {}.", httpProtocol.getMethod());
|
||||
return null;
|
||||
}
|
||||
|
||||
// params
|
||||
Map<String, String> params = httpProtocol.getParams();
|
||||
if (params != null && !params.isEmpty()) {
|
||||
@@ -306,7 +409,6 @@ public class HttpCollectImpl extends AbstractCollect {
|
||||
requestBuilder.addParameter(param.getKey(), param.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
// headers
|
||||
Map<String, String> headers = httpProtocol.getHeaders();
|
||||
if (headers != null && !headers.isEmpty()) {
|
||||
@@ -339,7 +441,7 @@ public class HttpCollectImpl extends AbstractCollect {
|
||||
|
||||
// 请求内容，会覆盖post协议的params
|
||||
if(StringUtils.hasLength(httpProtocol.getPayload())){
|
||||
requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(),"UTF-8"));
|
||||
requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(), StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
// uri
|
||||
|
||||
@@ -59,6 +59,10 @@ public interface DispatchConstants {
|
||||
* 解析方式 网站可用性监控规则 提供responseTime指标
|
||||
*/
|
||||
String PARSE_WEBSITE = "website";
|
||||
/**
|
||||
* 解析方式 网站地图全站可用性监控规则
|
||||
*/
|
||||
String PARSE_SITE_MAP = "sitemap";
|
||||
/**
|
||||
* 解析方式 prometheus规则
|
||||
*/
|
||||
|
||||
@@ -237,6 +237,7 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
|
||||
// 设置实例instance
|
||||
realValueRowBuilder.setInstance(instanceBuilder.toString());
|
||||
collectData.addValues(realValueRowBuilder.build());
|
||||
realValueRowBuilder.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,4 +8,10 @@ package com.usthe.collector.util;
|
||||
public interface CollectorConstants {
|
||||
|
||||
String RESPONSE_TIME = "responseTime";
|
||||
|
||||
String STATUS_CODE = "statusCode";
|
||||
|
||||
String ERROR_MSG = "errorMsg";
|
||||
|
||||
String URL = "url";
|
||||
}
|
||||
|
||||
49
manager/src/main/resources/define/app/fullsite.yml
Normal file
49
manager/src/main/resources/define/app/fullsite.yml
Normal file
@@ -0,0 +1,49 @@
|
||||
app: fullsite
|
||||
name:
|
||||
zh-CN: 全站监控
|
||||
en-US: FULL WEBSITE MONITOR
|
||||
configmap:
|
||||
- key: host
|
||||
type: 1
|
||||
- key: port
|
||||
type: 0
|
||||
- key: sitemap
|
||||
type: 1
|
||||
- key: ssl
|
||||
type: 1
|
||||
metrics:
|
||||
# 第一个监控指标组 cpu
|
||||
# 注意：内置监控指标有 (responseTime - 响应时间)
|
||||
- name: summary
|
||||
# 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
|
||||
# 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
|
||||
priority: 0
|
||||
# 指标组中的具体监控指标
|
||||
fields:
|
||||
# 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位
|
||||
- field: url
|
||||
type: 1
|
||||
instance: true
|
||||
- field: statusCode
|
||||
type: 1
|
||||
- field: responseTime
|
||||
type: 0
|
||||
unit: ms
|
||||
- field: errorMsg
|
||||
type: 1
|
||||
|
||||
# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
|
||||
protocol: http
|
||||
# 当protocol为http协议时具体的采集配置
|
||||
http:
|
||||
# 主机host: ipv4 ipv6 域名
|
||||
host: ^_^host^_^
|
||||
# 端口
|
||||
port: ^_^port^_^
|
||||
# url请求接口路径
|
||||
url: ^_^sitemap^_^
|
||||
# 请求方式 GET POST PUT DELETE PATCH
|
||||
method: GET
|
||||
# 是否启用ssl/tls,即是http还是https,默认false
|
||||
ssl: ^_^ssl^_^
|
||||
parseType: sitemap
|
||||
22
manager/src/main/resources/define/param/fullsite.yml
Normal file
22
manager/src/main/resources/define/param/fullsite.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
app: fullsite
|
||||
param:
|
||||
- field: host
|
||||
name: 主机Host
|
||||
type: host
|
||||
required: true
|
||||
- field: port
|
||||
name: 端口
|
||||
type: number
|
||||
range: '[0,65535]'
|
||||
required: true
|
||||
defaultValue: 80
|
||||
- field: sitemap
|
||||
name: 网站地图
|
||||
type: text
|
||||
limit: 200
|
||||
required: true
|
||||
placeholder: '网站SITEMAP地图地址 例如:/sitemap.xml'
|
||||
- field: ssl
|
||||
name: 启用HTTPS
|
||||
type: boolean
|
||||
required: true
|
||||
Reference in New Issue
Block a user