[manager]feature:[website api]monitor support keyword match (#72)

collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java

@@ -7,6 +7,7 @@ import com.google.gson.JsonParser;
 import com.usthe.collector.collect.AbstractCollect;
 import com.usthe.collector.collect.common.http.CommonHttpClient;
 import com.usthe.collector.dispatch.DispatchConstants;
+import com.usthe.collector.util.CollectUtil;
 import com.usthe.collector.util.CollectorConstants;
 import com.usthe.collector.util.JsonPathParser;
 import com.usthe.common.entity.job.Metrics;
@@ -100,7 +101,7 @@ public class HttpCollectImpl extends AbstractCollect {
                 String parseType = metrics.getHttp().getParseType();
                 try {
                     if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) {
-                        parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
+                        parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
                     } else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) {
                         parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
                     } else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
@@ -108,11 +109,11 @@ public class HttpCollectImpl extends AbstractCollect {
                     } else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) {
                         parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
                     } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){
-                        parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime);
+                        parseResponseByWebsite(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
                     } else if (DispatchConstants.PARSE_SITE_MAP.equals(parseType)) {
                         parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
                     } else {
-                        parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
+                        parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
                     }
                 } catch (Exception e) {
                     log.info("parse error: {}.", e.getMessage(), e);
@@ -169,13 +170,16 @@ public class HttpCollectImpl extends AbstractCollect {
         }
     }
 
-    private void parseResponseByWebsite(String resp, List<String> aliasFields,
+    private void parseResponseByWebsite(String resp, List<String> aliasFields, HttpProtocol http,
                                         CollectRep.MetricsData.Builder builder, Long responseTime) {
         CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
-        // todo resp 网站关键字监测
+        // 网站关键词数量监测
+        int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
         for (String alias : aliasFields) {
             if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
                 valueRowBuilder.addColumns(responseTime.toString());
+            } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
+                valueRowBuilder.addColumns(Integer.toString(keywordNum));
             } else {
                 valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
             }
@@ -277,6 +281,7 @@ public class HttpCollectImpl extends AbstractCollect {
     private void parseResponseByJsonPath(String resp, List<String> aliasFields, HttpProtocol http,
                                          CollectRep.MetricsData.Builder builder, Long responseTime) {
         List<Map<String, Object>> results = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript());
+        int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
         for (Map<String, Object> stringMap : results) {
             CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
             for (String alias : aliasFields) {
@@ -286,6 +291,8 @@ public class HttpCollectImpl extends AbstractCollect {
                 } else {
                     if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
                         valueRowBuilder.addColumns(responseTime.toString());
+                    } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
+                        valueRowBuilder.addColumns(Integer.toString(keywordNum));
                     } else {
                         valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
                     }
@@ -300,9 +307,10 @@ public class HttpCollectImpl extends AbstractCollect {
 
     }
 
-    private void parseResponseByDefault(String resp, List<String> aliasFields,
+    private void parseResponseByDefault(String resp, List<String> aliasFields, HttpProtocol http,
                                         CollectRep.MetricsData.Builder builder, Long responseTime) {
         JsonElement element = JsonParser.parseString(resp);
+        int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
         if (element.isJsonArray()) {
             JsonArray array = element.getAsJsonArray();
             for (JsonElement jsonElement : array) {
@@ -317,6 +325,8 @@ public class HttpCollectImpl extends AbstractCollect {
                         } else {
                             if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
                                 valueRowBuilder.addColumns(responseTime.toString());
+                            } else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
+                                valueRowBuilder.addColumns(Integer.toString(keywordNum));
                             } else {
                                 valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
                             }

collector/src/main/java/com/usthe/collector/util/CollectUtil.java

@@ -0,0 +1,35 @@
+package com.usthe.collector.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * 采集器工具类
+ * @author tom
+ * @date 2022/4/6 09:35
+ */
+public class CollectUtil {
+
+    /**
+     * 关键字匹配计数
+     * @param content 内容
+     * @param keyword 关键字
+     * @return 匹配次数
+     */
+    public static int countMatchKeyword(String content, String keyword) {
+        if (content == null || "".equals(content) || keyword == null || "".equals(keyword.trim())) {
+            return 0;
+        }
+        try {
+            Pattern pattern = Pattern.compile(keyword);
+            Matcher matcher = pattern.matcher(content);
+            int count = 0;
+            while (matcher.find()) {
+                count++;
+            }
+            return count;
+        } catch (Exception e) {
+            return 0;
+        }
+    }
+}

collector/src/main/java/com/usthe/collector/util/CollectorConstants.java

@@ -9,6 +9,8 @@ public interface CollectorConstants {
 
     String RESPONSE_TIME = "responseTime";
 
+    String KEYWORD = "keyword";
+
     String STATUS_CODE = "statusCode";
 
     String ERROR_MSG = "errorMsg";

common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java

@@ -49,7 +49,6 @@ public class HttpProtocol {
      * http请求携带的请求体
      */
     private String payload;
-
     /**
      * 认证信息
      */
@@ -66,6 +65,10 @@ public class HttpProtocol {
      * 数据解析脚本 当解析方式为 jsonPath or xmlPath时存在
      */
     private String parseScript;
+    /**
+     * 内容关键字
+     */
+    private String keyword;
 
     /**
      * 认证信息

manager/src/main/resources/define/app/api.yml

@@ -32,6 +32,8 @@ configmap:
     type: 3
   - key: params
     type: 3
+  - key: keyword
+    type: 1
 # 指标组列表
 metrics:
   # 第一个监控指标组 cpu
@@ -46,6 +48,9 @@ metrics:
       - field: responseTime
         type: 0
         unit: ms
+      - field: keyword
+        type: 0
+        unit: 次数
 # 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
     protocol: http
 # 当protocol为http协议时具体的采集配置
@@ -78,4 +83,5 @@ metrics:
         digestAuthPassword: ^_^password^_^
       # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控
       # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
-      parseType: website
+      parseType: website
+      keyword: ^_^keyword^_^

manager/src/main/resources/define/app/website.yml

@@ -18,10 +18,12 @@ configmap:
     type: 1
   - key: password
     type: 2
+  - key: keyword
+    type: 1
 # 指标组列表
 metrics:
   # 第一个监控指标组 cpu
-  # 注意：内置监控指标有 (responseTime - 响应时间)
+  # 注意：内置监控指标有 (responseTime - 响应时间, keyword - 关键字数量)
   - name: summary
     # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
     # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
@@ -32,6 +34,9 @@ metrics:
       - field: responseTime
         type: 0
         unit: ms
+      - field: keyword
+        type: 0
+        unit: 次数
 # 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
     protocol: http
 # 当protocol为http协议时具体的采集配置
@@ -55,4 +60,5 @@ metrics:
         digestAuthPassword: ^_^password^_^
       # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控
       # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
-      parseType: website
+      parseType: website
+      keyword: ^_^keyword^_^

manager/src/main/resources/define/param/api.yml

@@ -90,3 +90,8 @@ param:
     type: password
     required: false
     hide: true
+  - field: keyword
+    name: 关键字
+    type: text
+    required: false
+    hide: true

manager/src/main/resources/define/param/website.yml

@@ -51,4 +51,9 @@ param:
     name: 密码
     type: password
     required: false
+    hide: true
+  - field: keyword
+    name: 关键字
+    type: text
+    required: false
     hide: true