From e6673325a8bced7d1d520c18d324fe459a9562c8 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Tue, 15 Feb 2022 20:46:07 +0800 Subject: [PATCH] =?UTF-8?q?[collector,manager]=E6=94=AF=E6=8C=81sitemap?= =?UTF-8?q?=E5=85=A8=E7=AB=99=E7=B1=BB=E5=9E=8B=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../collect/http/HttpCollectImpl.java | 108 +++++++++++++++++- .../collector/dispatch/DispatchConstants.java | 4 + .../collector/dispatch/MetricsCollect.java | 1 + .../collector/util/CollectorConstants.java | 6 + .../main/resources/define/app/fullsite.yml | 49 ++++++++ .../main/resources/define/param/fullsite.yml | 22 ++++ 6 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 manager/src/main/resources/define/app/fullsite.yml create mode 100644 manager/src/main/resources/define/param/fullsite.yml diff --git a/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java b/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java index 769a31f..ce12603 100644 --- a/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java +++ b/collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java @@ -22,6 +22,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; import org.apache.http.client.protocol.HttpClientContext; @@ -31,13 +32,21 @@ import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import org.springframework.http.HttpMethod; import org.springframework.util.StringUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import javax.net.ssl.SSLException; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InterruptedIOException; import java.net.ConnectException; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -100,6 +109,8 @@ public class HttpCollectImpl extends AbstractCollect { parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){ parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime); + } else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) { + parseResponseBySiteMap(resp, metrics.getAliasFields(), builder); } else { parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime); } @@ -172,6 +183,99 @@ public class HttpCollectImpl extends AbstractCollect { builder.addValues(valueRowBuilder.build()); } + private void parseResponseBySiteMap(String resp, List aliasFields, + CollectRep.MetricsData.Builder builder) { + List siteUrls = new LinkedList<>(); + // 使用xml解析 + boolean isXmlFormat = true; + try { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + Document document = db.parse(new ByteArrayInputStream(resp.getBytes(StandardCharsets.UTF_8))); + NodeList urlList = document.getElementsByTagName("url"); + for (int i = 0; i < urlList.getLength(); i++) { + Node urlNode = urlList.item(i); + NodeList childNodes = urlNode.getChildNodes(); + for (int k = 0; k < childNodes.getLength(); k++) { + Node currentNode = childNodes.item(k); + // 区分出text类型的node以及element类型的node + if (currentNode.getNodeType() == Node.ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) { + //获取了loc节点的值 + siteUrls.add(currentNode.getFirstChild().getNodeValue()); + break; + } + } + } + } catch (Exception e) { + log.warn(e.getMessage(), e); + isXmlFormat = false; + } + // 若xml解析失败 用txt格式解析 + if (!isXmlFormat) { + try { + String[] urls = resp.split("\n"); + // 校验是否是URL + if (IpDomainUtil.isHasSchema(urls[0])) { + siteUrls.addAll(Arrays.asList(urls)); + } + } catch (Exception e) { + log.warn(e.getMessage(), e); + } + } + // todo siteUrl 限制数量 + // 开始循环访问每个site url 采集其 http status code, responseTime, 异常信息 + int maxUrlNum = 100; + for (String siteUrl : siteUrls) { + maxUrlNum --; + if (maxUrlNum <= 0) { + break; + } + String errorMsg = ""; + Integer statusCode = null; + long startTime = System.currentTimeMillis(); + try { + HttpGet httpGet = new HttpGet(siteUrl); + CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet); + statusCode = response.getStatusLine().getStatusCode(); + EntityUtils.consume(response.getEntity()); + } catch (ClientProtocolException e1) { + if (e1.getCause() != null) { + errorMsg = e1.getCause().getMessage(); + } else { + errorMsg = e1.getMessage(); + } + } catch (UnknownHostException e2) { + // 对端不可达 + errorMsg = "unknown host"; + } catch (InterruptedIOException | ConnectException | SSLException e3) { + // 对端连接失败 + errorMsg = "connect error: " + e3.getMessage(); + } catch (IOException e4) { + // 其它IO异常 + errorMsg = "io error: " + e4.getMessage(); + } catch (Exception e) { + errorMsg = "error: " + e.getMessage(); + } + long responseTime = System.currentTimeMillis() - startTime; + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String alias : aliasFields) { + if (CollectorConstants.URL.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(siteUrl); + } else if (CollectorConstants.STATUS_CODE.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(statusCode == null ? + CommonConstants.NULL_VALUE : String.valueOf(statusCode)); + } else if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(String.valueOf(responseTime)); + } else if (CollectorConstants.ERROR_MSG.equalsIgnoreCase(alias)) { + valueRowBuilder.addColumns(errorMsg); + } else { + valueRowBuilder.addColumns(CommonConstants.NULL_VALUE); + } + } + builder.addValues(valueRowBuilder.build()); + } + } + private void parseResponseByXmlPath(String resp, List aliasFields, HttpProtocol http, CollectRep.MetricsData.Builder builder) { } @@ -298,7 +402,6 @@ public class HttpCollectImpl extends AbstractCollect { log.error("not support the http method: {}.", httpProtocol.getMethod()); return null; } - // params Map params = httpProtocol.getParams(); if (params != null && !params.isEmpty()) { @@ -306,7 +409,6 @@ public class HttpCollectImpl extends AbstractCollect { requestBuilder.addParameter(param.getKey(), param.getValue()); } } - // headers Map headers = httpProtocol.getHeaders(); if (headers != null && !headers.isEmpty()) { @@ -339,7 +441,7 @@ public class HttpCollectImpl extends AbstractCollect { // 请求内容，会覆盖post协议的params if(StringUtils.hasLength(httpProtocol.getPayload())){ - requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(),"UTF-8")); + requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(), StandardCharsets.UTF_8)); } // uri diff --git a/collector/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java b/collector/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java index 4c85bf2..cc470c0 100644 --- a/collector/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java +++ b/collector/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java @@ -59,6 +59,10 @@ public interface DispatchConstants { * 解析方式 网站可用性监控规则 提供responseTime指标 */ String PARSE_WEBSITE = "website"; + /** + * 解析方式 网站地图全站可用性监控规则 + */ + String PARSE_SITE_MAP = "sitemap"; /** * 解析方式 prometheus规则 */ diff --git a/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java b/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java index c13e0f5..7209535 100644 --- a/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java +++ b/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java @@ -237,6 +237,7 @@ public class MetricsCollect implements Runnable, Comparable { // 设置实例instance realValueRowBuilder.setInstance(instanceBuilder.toString()); collectData.addValues(realValueRowBuilder.build()); + realValueRowBuilder.clear(); } } diff --git a/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java b/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java index 176acd3..3f3edbb 100644 --- a/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java +++ b/collector/src/main/java/com/usthe/collector/util/CollectorConstants.java @@ -8,4 +8,10 @@ package com.usthe.collector.util; public interface CollectorConstants { String RESPONSE_TIME = "responseTime"; + + String STATUS_CODE = "statusCode"; + + String ERROR_MSG = "errorMsg"; + + String URL = "url"; } diff --git a/manager/src/main/resources/define/app/fullsite.yml b/manager/src/main/resources/define/app/fullsite.yml new file mode 100644 index 0000000..8422a93 --- /dev/null +++ b/manager/src/main/resources/define/app/fullsite.yml @@ -0,0 +1,49 @@ +app: fullsite +name: + zh-CN: 全站监控 + en-US: FULL WEBSITE MONITOR +configmap: + - key: host + type: 1 + - key: port + type: 0 + - key: sitemap + type: 1 + - key: ssl + type: 1 +metrics: + # 第一个监控指标组 cpu + # 注意：内置监控指标有 (responseTime - 响应时间) + - name: summary + # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集 + # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度 + priority: 0 + # 指标组中的具体监控指标 + fields: + # 指标信息 包括 field名称 type字段类型:0-number数字,1-string字符串 instance是否为实例主键 unit:指标单位 + - field: url + type: 1 + instance: true + - field: statusCode + type: 1 + - field: responseTime + type: 0 + unit: ms + - field: errorMsg + type: 1 + +# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http +# 当protocol为http协议时具体的采集配置 + http: + # 主机host: ipv4 ipv6 域名 + host: ^_^host^_^ + # 端口 + port: ^_^port^_^ + # url请求接口路径 + url: ^_^sitemap^_^ + # 请求方式 GET POST PUT DELETE PATCH + method: GET + # 是否启用ssl/tls,即是http还是https,默认false + ssl: ^_^ssl^_^ + parseType: sitemap \ No newline at end of file diff --git a/manager/src/main/resources/define/param/fullsite.yml b/manager/src/main/resources/define/param/fullsite.yml new file mode 100644 index 0000000..71ef602 --- /dev/null +++ b/manager/src/main/resources/define/param/fullsite.yml @@ -0,0 +1,22 @@ +app: fullsite +param: + - field: host + name: 主机Host + type: host + required: true + - field: port + name: 端口 + type: number + range: '[0,65535]' + required: true + defaultValue: 80 + - field: sitemap + name: 网站地图 + type: text + limit: 200 + required: true + placeholder: '网站SITEMAP地图地址 例如:/sitemap.xml' + - field: ssl + name: 启用HTTPS + type: boolean + required: true \ No newline at end of file