[collector,manager]支持sitemap全站类型监控

collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java

@@ -22,6 +22,7 @@ import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.methods.RequestBuilder;
 import org.apache.http.client.protocol.HttpClientContext;
@@ -31,13 +32,21 @@ import org.apache.http.protocol.HttpContext;
 import org.apache.http.util.EntityUtils;
 import org.springframework.http.HttpMethod;
 import org.springframework.util.StringUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 
 import javax.net.ssl.SSLException;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.ConnectException;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -100,6 +109,8 @@ public class HttpCollectImpl extends AbstractCollect {
                         parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
                     } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){
                         parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime);
+                    } else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) {
+                        parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
                     } else {
                         parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
                     }
@@ -172,6 +183,99 @@ public class HttpCollectImpl extends AbstractCollect {
         builder.addValues(valueRowBuilder.build());
     }
 
+    private void parseResponseBySiteMap(String resp, List<String> aliasFields,
+                                        CollectRep.MetricsData.Builder builder) {
+        List<String> siteUrls = new LinkedList<>();
+        // 使用xml解析
+        boolean isXmlFormat = true;
+        try {
+            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+            DocumentBuilder db = dbf.newDocumentBuilder();
+            Document document = db.parse(new ByteArrayInputStream(resp.getBytes(StandardCharsets.UTF_8)));
+            NodeList urlList = document.getElementsByTagName("url");
+            for (int i = 0; i < urlList.getLength(); i++) {
+                Node urlNode = urlList.item(i);
+                NodeList childNodes = urlNode.getChildNodes();
+                for (int k = 0; k < childNodes.getLength(); k++) {
+                    Node currentNode = childNodes.item(k);
+                    // 区分出text类型的node以及element类型的node
+                    if (currentNode.getNodeType() == Node.ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
+                        //获取了loc节点的值
+                        siteUrls.add(currentNode.getFirstChild().getNodeValue());
+                        break;
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.warn(e.getMessage(), e);
+            isXmlFormat = false;
+        }
+        // 若xml解析失败 用txt格式解析
+        if (!isXmlFormat) {
+            try {
+                String[] urls = resp.split("\n");
+                // 校验是否是URL
+                if (IpDomainUtil.isHasSchema(urls[0])) {
+                    siteUrls.addAll(Arrays.asList(urls));
+                }
+            } catch (Exception e) {
+                log.warn(e.getMessage(), e);
+            }
+        }
+        // todo siteUrl 限制数量
+        // 开始循环访问每个site url 采集其 http status code, responseTime, 异常信息
+        int maxUrlNum = 100;
+        for (String siteUrl : siteUrls) {
+            maxUrlNum --;
+            if (maxUrlNum <= 0) {
+                break;
+            }
+            String errorMsg = "";
+            Integer statusCode = null;
+            long startTime = System.currentTimeMillis();
+            try {
+                HttpGet httpGet = new HttpGet(siteUrl);
+                CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet);
+                statusCode = response.getStatusLine().getStatusCode();
+                EntityUtils.consume(response.getEntity());
+            } catch (ClientProtocolException e1) {
+                if (e1.getCause() != null) {
+                    errorMsg = e1.getCause().getMessage();
+                } else {
+                    errorMsg = e1.getMessage();
+                }
+            } catch (UnknownHostException e2) {
+                // 对端不可达
+                errorMsg = "unknown host";
+            } catch (InterruptedIOException | ConnectException | SSLException e3) {
+                // 对端连接失败
+                errorMsg = "connect error: " + e3.getMessage();
+            } catch (IOException e4) {
+                // 其它IO异常
+                errorMsg = "io error: " + e4.getMessage();
+            } catch (Exception e) {
+                errorMsg = "error: " + e.getMessage();
+            }
+            long responseTime = System.currentTimeMillis() - startTime;
+            CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
+            for (String alias : aliasFields) {
+                if (CollectorConstants.URL.equalsIgnoreCase(alias)) {
+                    valueRowBuilder.addColumns(siteUrl);
+                } else if (CollectorConstants.STATUS_CODE.equalsIgnoreCase(alias)) {
+                    valueRowBuilder.addColumns(statusCode == null ?
+                            CommonConstants.NULL_VALUE : String.valueOf(statusCode));
+                } else if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                    valueRowBuilder.addColumns(String.valueOf(responseTime));
+                } else if (CollectorConstants.ERROR_MSG.equalsIgnoreCase(alias)) {
+                    valueRowBuilder.addColumns(errorMsg);
+                } else {
+                    valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                }
+            }
+            builder.addValues(valueRowBuilder.build());
+        }
+    }
+
     private void parseResponseByXmlPath(String resp, List<String> aliasFields, HttpProtocol http,
                                         CollectRep.MetricsData.Builder builder) {
     }
@@ -298,7 +402,6 @@ public class HttpCollectImpl extends AbstractCollect {
             log.error("not support the http method: {}.", httpProtocol.getMethod());
             return null;
         }
-
         // params
         Map<String, String> params = httpProtocol.getParams();
         if (params != null && !params.isEmpty()) {
@@ -306,7 +409,6 @@ public class HttpCollectImpl extends AbstractCollect {
                 requestBuilder.addParameter(param.getKey(), param.getValue());
             }
         }
-
         // headers
         Map<String, String> headers = httpProtocol.getHeaders();
         if (headers != null && !headers.isEmpty()) {
@@ -339,7 +441,7 @@ public class HttpCollectImpl extends AbstractCollect {
 
         // 请求内容，会覆盖post协议的params
         if(StringUtils.hasLength(httpProtocol.getPayload())){
-            requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(),"UTF-8"));
+            requestBuilder.setEntity(new StringEntity(httpProtocol.getPayload(), StandardCharsets.UTF_8));
         }
 
         // uri

collector/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java

@@ -59,6 +59,10 @@ public interface DispatchConstants {
      * 解析方式 网站可用性监控规则 提供responseTime指标
      */
     String PARSE_WEBSITE = "website";
+    /**
+     * 解析方式 网站地图全站可用性监控规则
+     */
+    String PARSE_SITE_MAP = "sitemap";
     /**
      * 解析方式 prometheus规则
      */

collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java

@@ -237,6 +237,7 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
             // 设置实例instance
             realValueRowBuilder.setInstance(instanceBuilder.toString());
             collectData.addValues(realValueRowBuilder.build());
+            realValueRowBuilder.clear();
         }
     }
 

collector/src/main/java/com/usthe/collector/util/CollectorConstants.java

@@ -8,4 +8,10 @@ package com.usthe.collector.util;
 public interface CollectorConstants {
 
     String RESPONSE_TIME = "responseTime";
+
+    String STATUS_CODE = "statusCode";
+
+    String ERROR_MSG = "errorMsg";
+
+    String URL = "url";
 }

manager/src/main/resources/define/app/fullsite.yml

@@ -0,0 +1,49 @@
+app: fullsite
+name:
+  zh-CN: 全站监控
+  en-US: FULL WEBSITE MONITOR
+configmap:
+  - key: host
+    type: 1
+  - key: port
+    type: 0
+  - key: sitemap
+    type: 1
+  - key: ssl
+    type: 1
+metrics:
+  # 第一个监控指标组 cpu
+  # 注意：内置监控指标有 (responseTime - 响应时间)
+  - name: summary
+    # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
+    # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
+    priority: 0
+    # 指标组中的具体监控指标
+    fields:
+      # 指标信息 包括 field名称   type字段类型:0-number数字,1-string字符串   instance是否为实例主键   unit:指标单位
+      - field: url
+        type: 1
+        instance: true
+      - field: statusCode
+        type: 1
+      - field: responseTime
+        type: 0
+        unit: ms
+      - field: errorMsg
+        type: 1
+
+# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
+    protocol: http
+# 当protocol为http协议时具体的采集配置
+    http:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      # url请求接口路径
+      url: ^_^sitemap^_^
+      # 请求方式 GET POST PUT DELETE PATCH
+      method: GET
+      # 是否启用ssl/tls,即是http还是https,默认false
+      ssl: ^_^ssl^_^
+      parseType: sitemap

manager/src/main/resources/define/param/fullsite.yml

@@ -0,0 +1,22 @@
+app: fullsite
+param:
+  - field: host
+    name: 主机Host
+    type: host
+    required: true
+  - field: port
+    name: 端口
+    type: number
+    range: '[0,65535]'
+    required: true
+    defaultValue: 80
+  - field: sitemap
+    name: 网站地图
+    type: text
+    limit: 200
+    required: true
+    placeholder: '网站SITEMAP地图地址 例如:/sitemap.xml'
+  - field: ssl
+    name: 启用HTTPS
+    type: boolean
+    required: true