[monitor] 支持网站监控类型，降低雪花算法UUID最大值解决ts json解析大数问题

collector/server/src/main/java/com/usthe/collector/Collector.java

@@ -5,10 +5,6 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;
 
-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.InetAddress;
-
 /**
  * collector start
  * @author tomsun28
@@ -19,20 +15,5 @@ import java.net.InetAddress;
 public class Collector {
     public static void main(String[] args) {
         SpringApplication.run(Collector.class, args);
-//        DatagramSocket s = new DatagramSocket();
-//
-//        /** 2、提供数据，封装打包  ---DatagramPacket(byte[] buf, int length, InetAddress address, int port)  */
-//
-//        byte[] bs = "正在使用UDP发送--我是数据! ".getBytes();
-//        DatagramPacket dp = new DatagramPacket(bs, bs.length, InetAddress.getByName("192.168.1.189"), 8070);
-//
-//        /** 3、使用send发送 */
-//        try {
-//            s.send(dp);
-//            s.receive(dp)
-//        } catch (IOException e) {
-//            System.out.println("发送失败： ");
-//            e.printStackTrace();
-//        }
     }
 }

collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java

@@ -7,6 +7,7 @@ import com.google.gson.JsonParser;
 import com.usthe.collector.collect.AbstractCollect;
 import com.usthe.collector.common.http.HttpClientPool;
 import com.usthe.collector.dispatch.DispatchConstants;
+import com.usthe.collector.util.CollectorConstants;
 import com.usthe.collector.util.JsonPathParser;
 import com.usthe.common.entity.job.Metrics;
 import com.usthe.common.entity.job.protocol.HttpProtocol;
@@ -56,6 +57,7 @@ public class HttpCollectImpl extends AbstractCollect {
     @Override
     public void collect(CollectRep.MetricsData.Builder builder,
                         long appId, String app, Metrics metrics) {
+        long startTime = System.currentTimeMillis();
         // 简单校验必有参数
         if (metrics == null || metrics.getHttp() == null) {
             builder.setCode(CollectRep.Code.FAIL);
@@ -84,18 +86,21 @@ public class HttpCollectImpl extends AbstractCollect {
                     builder.setMsg("statusCode: " + statusCode + ",entity empty.");
                     return;
                 }
+                Long responseTime  = System.currentTimeMillis() - startTime;
                 String parseType = metrics.getHttp().getParseType();
                 try {
                     if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) {
-                        parseResponseByDefault(resp, metrics.getAliasFields(), builder);
+                        parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
                     } else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) {
-                        parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
+                        parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
                     } else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
                         parseResponseByPrometheus(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
                     } else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) {
                         parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
+                    } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){
+                        parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime);
                     } else {
-                        parseResponseByDefault(resp, metrics.getAliasFields(), builder);
+                        parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
                     }
                 } catch (Exception e) {
                     log.info("parse error: {}.", e.getMessage(), e);
@@ -140,13 +145,26 @@ public class HttpCollectImpl extends AbstractCollect {
         }
     }
 
+    private void parseResponseByWebsite(String resp, List<String> aliasFields,
+                                        CollectRep.MetricsData.Builder builder, Long responseTime) {
+        CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
+        // todo resp 网站关键字监测
+        for (String alias : aliasFields) {
+            if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                valueRowBuilder.addColumns(responseTime.toString());
+            } else {
+                valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+            }
+        }
+        builder.addValues(valueRowBuilder.build());
+    }
+
     private void parseResponseByXmlPath(String resp, List<String> aliasFields, HttpProtocol http,
                                         CollectRep.MetricsData.Builder builder) {
-
     }
 
     private void parseResponseByJsonPath(String resp, List<String> aliasFields, HttpProtocol http,
-                                         CollectRep.MetricsData.Builder builder) {
+                                         CollectRep.MetricsData.Builder builder, Long responseTime) {
         List<Map<String, Object>> results = JsonPathParser.parseContentWithJsonPath(resp,http. getParseScript());
         for (Map<String, Object> stringMap : results) {
             CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
@@ -155,7 +173,11 @@ public class HttpCollectImpl extends AbstractCollect {
                 if (value != null) {
                     valueRowBuilder.addColumns(String.valueOf(value));
                 } else {
-                    valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                    if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                        valueRowBuilder.addColumns(responseTime.toString());
+                    } else {
+                        valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                    }
                 }
             }
             builder.addValues(valueRowBuilder.build());
@@ -167,7 +189,8 @@ public class HttpCollectImpl extends AbstractCollect {
 
     }
 
-    private void parseResponseByDefault(String resp, List<String> aliasFields, CollectRep.MetricsData.Builder builder) {
+    private void parseResponseByDefault(String resp, List<String> aliasFields,
+                                        CollectRep.MetricsData.Builder builder, Long responseTime) {
         JsonElement element = JsonParser.parseString(resp);
         if (element.isJsonArray()) {
             JsonArray array = element.getAsJsonArray();
@@ -181,7 +204,11 @@ public class HttpCollectImpl extends AbstractCollect {
                             String value = valueElement.getAsString();
                             valueRowBuilder.addColumns(value);
                         } else {
-                            valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                            if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                                valueRowBuilder.addColumns(responseTime.toString());
+                            } else {
+                                valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                            }
                         }
                     }
                     builder.addValues(valueRowBuilder.build());
@@ -283,6 +310,8 @@ public class HttpCollectImpl extends AbstractCollect {
         } else if (DispatchConstants.PARSE_PROMETHEUS.equals(httpProtocol.getParseType())) {
             requestBuilder.addHeader(HttpHeaders.ACCEPT, DispatchConstants.PARSE_PROMETHEUS_ACCEPT);
             requestBuilder.addHeader(HttpHeaders.ACCEPT_ENCODING, "gzip");
+        } else {
+            requestBuilder.addHeader(HttpHeaders.ACCEPT, "*/*");
         }
 
         // 判断是否使用Bearer Token认证

collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java

@@ -57,6 +57,10 @@ public interface DispatchConstants {
      */
     String PARSE_XML_PATH = "xmlPath";
     /**
+     * 解析方式 网站可用性监控规则 提供responseTime指标
+     */
+    String PARSE_WEBSITE = "website";
+    /**
      * 解析方式 prometheus规则
      */
     String PARSE_PROMETHEUS = "prometheus";

collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java

@@ -0,0 +1,11 @@
+package com.usthe.collector.util;
+
+/**
+ * collector 常量
+ * @author tom
+ * @date 2021/12/3 12:15
+ */
+public interface CollectorConstants {
+
+    String RESPONSE_TIME = "responseTime";
+}

common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java

@@ -10,7 +10,7 @@ public class SnowFlakeIdGenerator {
     private final static SnowFlakeIdWorker ID_WORKER;
 
     static {
-        ID_WORKER = new SnowFlakeIdWorker(1, 0);
+        ID_WORKER = new SnowFlakeIdWorker(0);
     }
 
     public static long generateId() {

common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java

@@ -2,6 +2,7 @@ package com.usthe.common.util;
 
 /**
  * 雪花算法生成器实例
+ * 注意 由于前端JS TS 在json解析大数会造成精度丢失 UUID 不能超过 9007199254740991（16位）
  * @author from https://www.cnblogs.com/vchar/p/14857677.html
  * @date 2021/11/10 10:58
  */
@@ -15,12 +16,12 @@ public class SnowFlakeIdWorker {
     /**
      * 机器 ID 所占的位数
      */
-    private static final long WORKER_ID_BITS = 5L;
+    private static final long WORKER_ID_BITS = 2L;
 
     /**
      * 数据标识 ID 所占的位数
      */
-    private static final long DATA_CENTER_ID_BITS = 5L;
+    private static final long DATA_CENTER_ID_BITS = 4L;
 
     /**
      * 支持的最大机器ID，最大为31

common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java

@@ -0,0 +1,22 @@
+package com.usthe.common.util;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author tom
+ * @date 2021/12/3 13:28
+ */
+class SnowFlakeIdGeneratorTest {
+
+    @Test
+    void generateId() {
+        // 注意 由于前端JS TS 在json解析大数会造成精度丢失 UUID 不能超过 9007199254740991（16位）
+        for (int i = 0; i < 1000; i++) {
+            long id = SnowFlakeIdGenerator.generateId();
+            Assertions.assertTrue(id < 9007199254740991L);
+        }
+    }
+}

manager/src/main/resources/define/app/A-example.yml

@@ -13,7 +13,8 @@ configmap:
     type: 2
 # 指标组列表
 metrics:
-  # 第一个监控指标组 cpu
+# 第一个监控指标组 cpu
+# 注意：内置监控指标有 (responseTime - 响应时间)
   - name: cpu
     # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
     # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
@@ -74,7 +75,8 @@ metrics:
         type: Basic Auth
         basicAuthUsername: ^_^username^_^
         basicAuthPassword: ^_^password^_^
-      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
+      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控
+      # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
       parseType: jsonPath
       parseScript: '$'
 

manager/src/main/resources/define/app/api.yml

@@ -0,0 +1,56 @@
+# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
+app: api
+# 参数映射map. type是参数类型: 0-number数字, 1-string明文字符串, 2-secret加密字符串
+# 强制固定必须参数 - host
+configmap:
+  - key: host
+    type: 1
+  - key: port
+    type: 0
+  - key: uri
+    type: 1
+  - key: method
+    type: 1
+  - key: ssl
+    type: 1
+  - key: username
+    type: 1
+  - key: password
+    type: 2
+# 指标组列表
+metrics:
+  # 第一个监控指标组 cpu
+  # 注意：内置监控指标有 (responseTime - 响应时间)
+  - name: summary
+    # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
+    # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
+    priority: 0
+    # 指标组中的具体监控指标
+    fields:
+      # 指标信息 包括 field名称   type字段类型:0-number数字,1-string字符串   instance是否为实例主键   unit:指标单位
+      - field: responseTime
+        type: 0
+        unit: ms
+# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
+    protocol: http
+# 当protocol为http协议时具体的采集配置
+    http:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      # url请求接口路径
+      url: ^_^uri^_^
+      # 请求方式 GET POST PUT DELETE PATCH
+      method: ^_^method^_^
+      # 是否启用ssl/tls,即是http还是https,默认false
+      ssl: false
+      # 认证
+      authorization:
+        # 认证方式: Basic Auth, Digest Auth, Bearer Token
+        type: Basic Auth
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控
+      # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
+      parseType: website

manager/src/main/resources/define/param/api.yml

@@ -0,0 +1,47 @@
+# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
+app: api
+# 强制固定必须参数 - host(ipv4,ipv6,域名)
+param:
+    # field-字段名称标识符
+  - field: host
+    # name-参数字段显示名称
+    name: 主机Host
+    # type-字段类型,样式(大部分映射input标签type属性)
+    type: host
+    # 是否是必输项 true-必填 false-可选
+    required: true
+  - field: port
+    name: 端口
+    type: number
+    # 当type为number时,用range表示范围
+    range: '[0,65535]'
+    required: true
+  - field: uri
+    name: URI路径
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 100
+    required: true
+  - field: method
+    name: 请求方式
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 20
+    required: true
+  - field: username
+    name: 用户名
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 20
+    required: false
+  - field: password
+    name: 密码
+    type: password
+    required: false
+  - field: ssl
+    name: 使用SSL
+    type: boolean
+    required: true
+    # 当type为boolean时,前端用switch展示开关
+    # 当type为radio单选框,checkbox复选框时,option表示可选项值列表
+    # option: Yes,No

manager/src/main/resources/sureness.yml

@@ -10,6 +10,7 @@ resourceRole:
 # load api resource which do not need be protected, means them need be excluded.
 # these api resource can be access by everyone
 excludedResource:
+  - /**===*
   - /account/auth/form===post
   - /**/*.html===get
   - /**/*.js===get

web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts

@@ -102,7 +102,7 @@ export class MonitorNewComponent implements OnInit {
       "params": this.params
     };
     this.isSpinning = true;
-    this.monitorSvc.newMonitor(detectMonitor)
+    this.monitorSvc.detectMonitor(detectMonitor)
       .subscribe(message => {
         this.isSpinning = false;
         if (message.code === 0) {

web-app/src/assets/tmp/app-data.json

@@ -35,8 +35,8 @@
           "icon": "anticon-cloud",
           "children": [
             {
-              "text": "http",
-              "link": "/monitors?app=http",
+              "text": "api",
+              "link": "/monitors?app=api",
               "i18n": "monitor.app.http"
             },
             {

web-app/src/assets/tmp/i18n/zh-CN.json

@@ -28,6 +28,7 @@
     },
     "app": {
       "": "监控类型",
+      "api": "HTTP API",
       "http": "HTTP API",
       "ping": "PING连通性",
       "telnet": "Telnet端口可用性",