[monitor] 支持网站监控类型，降低雪花算法UUID最大值解决ts json解析大数问题

2021-12-03 13:41:17 +08:00
parent e7e2e87e11
commit 4ae1145d19
14 changed files with 190 additions and 35 deletions
--- a/collector/server/src/main/java/com/usthe/collector/Collector.java
+++ b/collector/server/src/main/java/com/usthe/collector/Collector.java
@@ -5,10 +5,6 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;

-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.InetAddress;
-
 /**
 * collector start
 * @author tomsun28
@@ -19,20 +15,5 @@ import java.net.InetAddress;
 public class Collector {
    public static void main(String[] args) {
        SpringApplication.run(Collector.class, args);
-//        DatagramSocket s = new DatagramSocket();
-//
-//        /** 2、提供数据，封装打包  ---DatagramPacket(byte[] buf, int length, InetAddress address, int port)  */
-//
-//        byte[] bs = "正在使用UDP发送--我是数据! ".getBytes();
-//        DatagramPacket dp = new DatagramPacket(bs, bs.length, InetAddress.getByName("192.168.1.189"), 8070);
-//
-//        /** 3、使用send发送 */
-//        try {
-//            s.send(dp);
-//            s.receive(dp)
-//        } catch (IOException e) {
-//            System.out.println("发送失败： ");
-//            e.printStackTrace();
-//        }
    }
 }
--- a/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java
+++ b/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java
@@ -7,6 +7,7 @@ import com.google.gson.JsonParser;
 import com.usthe.collector.collect.AbstractCollect;
 import com.usthe.collector.common.http.HttpClientPool;
 import com.usthe.collector.dispatch.DispatchConstants;
+import com.usthe.collector.util.CollectorConstants;
 import com.usthe.collector.util.JsonPathParser;
 import com.usthe.common.entity.job.Metrics;
 import com.usthe.common.entity.job.protocol.HttpProtocol;
@@ -56,6 +57,7 @@ public class HttpCollectImpl extends AbstractCollect {
    @Override
    public void collect(CollectRep.MetricsData.Builder builder,
                        long appId, String app, Metrics metrics) {
+        long startTime = System.currentTimeMillis();
        // 简单校验必有参数
        if (metrics == null || metrics.getHttp() == null) {
            builder.setCode(CollectRep.Code.FAIL);
@@ -84,18 +86,21 @@ public class HttpCollectImpl extends AbstractCollect {
                    builder.setMsg("statusCode: " + statusCode + ",entity empty.");
                    return;
                }
+                Long responseTime  = System.currentTimeMillis() - startTime;
                String parseType = metrics.getHttp().getParseType();
                try {
                    if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) {
-                        parseResponseByDefault(resp, metrics.getAliasFields(), builder);
+                        parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
                    } else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) {
-                        parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
+                        parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
                    } else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
                        parseResponseByPrometheus(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
                    } else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) {
                        parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
+                    } else if (DispatchConstants.PARSE_WEBSITE.equals(parseType)){
+                        parseResponseByWebsite(resp, metrics.getAliasFields(), builder, responseTime);
                    } else {
-                        parseResponseByDefault(resp, metrics.getAliasFields(), builder);
+                        parseResponseByDefault(resp, metrics.getAliasFields(), builder, responseTime);
                    }
                } catch (Exception e) {
                    log.info("parse error: {}.", e.getMessage(), e);
@@ -140,13 +145,26 @@ public class HttpCollectImpl extends AbstractCollect {
        }
    }

+    private void parseResponseByWebsite(String resp, List<String> aliasFields,
+                                        CollectRep.MetricsData.Builder builder, Long responseTime) {
+        CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
+        // todo resp 网站关键字监测
+        for (String alias : aliasFields) {
+            if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                valueRowBuilder.addColumns(responseTime.toString());
+            } else {
+                valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+            }
+        }
+        builder.addValues(valueRowBuilder.build());
+    }
+
    private void parseResponseByXmlPath(String resp, List<String> aliasFields, HttpProtocol http,
                                        CollectRep.MetricsData.Builder builder) {
-
    }

    private void parseResponseByJsonPath(String resp, List<String> aliasFields, HttpProtocol http,
-                                         CollectRep.MetricsData.Builder builder) {
+                                         CollectRep.MetricsData.Builder builder, Long responseTime) {
        List<Map<String, Object>> results = JsonPathParser.parseContentWithJsonPath(resp,http. getParseScript());
        for (Map<String, Object> stringMap : results) {
            CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
@@ -155,7 +173,11 @@ public class HttpCollectImpl extends AbstractCollect {
                if (value != null) {
                    valueRowBuilder.addColumns(String.valueOf(value));
                } else {
-                    valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                    if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                        valueRowBuilder.addColumns(responseTime.toString());
+                    } else {
+                        valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                    }
                }
            }
            builder.addValues(valueRowBuilder.build());
@@ -167,7 +189,8 @@ public class HttpCollectImpl extends AbstractCollect {

    }

-    private void parseResponseByDefault(String resp, List<String> aliasFields, CollectRep.MetricsData.Builder builder) {
+    private void parseResponseByDefault(String resp, List<String> aliasFields,
+                                        CollectRep.MetricsData.Builder builder, Long responseTime) {
        JsonElement element = JsonParser.parseString(resp);
        if (element.isJsonArray()) {
            JsonArray array = element.getAsJsonArray();
@@ -181,7 +204,11 @@ public class HttpCollectImpl extends AbstractCollect {
                            String value = valueElement.getAsString();
                            valueRowBuilder.addColumns(value);
                        } else {
-                            valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                            if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
+                                valueRowBuilder.addColumns(responseTime.toString());
+                            } else {
+                                valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
+                            }
                        }
                    }
                    builder.addValues(valueRowBuilder.build());
@@ -283,6 +310,8 @@ public class HttpCollectImpl extends AbstractCollect {
        } else if (DispatchConstants.PARSE_PROMETHEUS.equals(httpProtocol.getParseType())) {
            requestBuilder.addHeader(HttpHeaders.ACCEPT, DispatchConstants.PARSE_PROMETHEUS_ACCEPT);
            requestBuilder.addHeader(HttpHeaders.ACCEPT_ENCODING, "gzip");
+        } else {
+            requestBuilder.addHeader(HttpHeaders.ACCEPT, "*/*");
        }

        // 判断是否使用Bearer Token认证
--- a/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java
+++ b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java
@@ -56,6 +56,10 @@ public interface DispatchConstants {
     * 解析方式 自定义xml path
     */
    String PARSE_XML_PATH = "xmlPath";
+    /**
+     * 解析方式 网站可用性监控规则 提供responseTime指标
+     */
+    String PARSE_WEBSITE = "website";
    /**
     * 解析方式 prometheus规则
     */
--- a/collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java
+++ b/collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java
@@ -0,0 +1,11 @@
+package com.usthe.collector.util;
+
+/**
+ * collector 常量
+ * @author tom
+ * @date 2021/12/3 12:15
+ */
+public interface CollectorConstants {
+
+    String RESPONSE_TIME = "responseTime";
+}
--- a/common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java
+++ b/common/src/main/java/com/usthe/common/util/SnowFlakeIdGenerator.java
@@ -10,7 +10,7 @@ public class SnowFlakeIdGenerator {
    private final static SnowFlakeIdWorker ID_WORKER;

    static {
-        ID_WORKER = new SnowFlakeIdWorker(1, 0);
+        ID_WORKER = new SnowFlakeIdWorker(0);
    }

    public static long generateId() {
--- a/common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java
+++ b/common/src/main/java/com/usthe/common/util/SnowFlakeIdWorker.java
@@ -2,6 +2,7 @@ package com.usthe.common.util;

 /**
 * 雪花算法生成器实例
+ * 注意 由于前端JS TS 在json解析大数会造成精度丢失 UUID 不能超过 9007199254740991（16位）
 * @author from https://www.cnblogs.com/vchar/p/14857677.html
 * @date 2021/11/10 10:58
 */
@@ -15,12 +16,12 @@ public class SnowFlakeIdWorker {
    /**
     * 机器 ID 所占的位数
     */
-    private static final long WORKER_ID_BITS = 5L;
+    private static final long WORKER_ID_BITS = 2L;

    /**
     * 数据标识 ID 所占的位数
     */
-    private static final long DATA_CENTER_ID_BITS = 5L;
+    private static final long DATA_CENTER_ID_BITS = 4L;

    /**
     * 支持的最大机器ID，最大为31
--- a/common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java
+++ b/common/src/test/java/com/usthe/common/util/SnowFlakeIdGeneratorTest.java
@@ -0,0 +1,22 @@
+package com.usthe.common.util;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @author tom
+ * @date 2021/12/3 13:28
+ */
+class SnowFlakeIdGeneratorTest {
+
+    @Test
+    void generateId() {
+        // 注意 由于前端JS TS 在json解析大数会造成精度丢失 UUID 不能超过 9007199254740991（16位）
+        for (int i = 0; i < 1000; i++) {
+            long id = SnowFlakeIdGenerator.generateId();
+            Assertions.assertTrue(id < 9007199254740991L);
+        }
+    }
+}
--- a/manager/src/main/resources/define/app/A-example.yml
+++ b/manager/src/main/resources/define/app/A-example.yml
@@ -13,7 +13,8 @@ configmap:
    type: 2
 # 指标组列表
 metrics:
-  # 第一个监控指标组 cpu
+# 第一个监控指标组 cpu
+# 注意：内置监控指标有 (responseTime - 响应时间)
  - name: cpu
    # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
    # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
@@ -74,7 +75,8 @@ metrics:
        type: Basic Auth
        basicAuthUsername: ^_^username^_^
        basicAuthPassword: ^_^password^_^
-      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
+      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控
+      # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
      parseType: jsonPath
      parseScript: '$'

--- a/manager/src/main/resources/define/app/api.yml
+++ b/manager/src/main/resources/define/app/api.yml
@@ -0,0 +1,56 @@
+# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
+app: api
+# 参数映射map. type是参数类型: 0-number数字, 1-string明文字符串, 2-secret加密字符串
+# 强制固定必须参数 - host
+configmap:
+  - key: host
+    type: 1
+  - key: port
+    type: 0
+  - key: uri
+    type: 1
+  - key: method
+    type: 1
+  - key: ssl
+    type: 1
+  - key: username
+    type: 1
+  - key: password
+    type: 2
+# 指标组列表
+metrics:
+  # 第一个监控指标组 cpu
+  # 注意：内置监控指标有 (responseTime - 响应时间)
+  - name: summary
+    # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
+    # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
+    priority: 0
+    # 指标组中的具体监控指标
+    fields:
+      # 指标信息 包括 field名称   type字段类型:0-number数字,1-string字符串   instance是否为实例主键   unit:指标单位
+      - field: responseTime
+        type: 0
+        unit: ms
+# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
+    protocol: http
+# 当protocol为http协议时具体的采集配置
+    http:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      # url请求接口路径
+      url: ^_^uri^_^
+      # 请求方式 GET POST PUT DELETE PATCH
+      method: ^_^method^_^
+      # 是否启用ssl/tls,即是http还是https,默认false
+      ssl: false
+      # 认证
+      authorization:
+        # 认证方式: Basic Auth, Digest Auth, Bearer Token
+        type: Basic Auth
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-网站可用性指标监控
+      # todo xmlPath-xmlPath脚本,prometheus-Prometheus数据规则
+      parseType: website
--- a/manager/src/main/resources/define/param/api.yml
+++ b/manager/src/main/resources/define/param/api.yml
@@ -0,0 +1,47 @@
+# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
+app: api
+# 强制固定必须参数 - host(ipv4,ipv6,域名)
+param:
+    # field-字段名称标识符
+  - field: host
+    # name-参数字段显示名称
+    name: 主机Host
+    # type-字段类型,样式(大部分映射input标签type属性)
+    type: host
+    # 是否是必输项 true-必填 false-可选
+    required: true
+  - field: port
+    name: 端口
+    type: number
+    # 当type为number时,用range表示范围
+    range: '[0,65535]'
+    required: true
+  - field: uri
+    name: URI路径
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 100
+    required: true
+  - field: method
+    name: 请求方式
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 20
+    required: true
+  - field: username
+    name: 用户名
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 20
+    required: false
+  - field: password
+    name: 密码
+    type: password
+    required: false
+  - field: ssl
+    name: 使用SSL
+    type: boolean
+    required: true
+    # 当type为boolean时,前端用switch展示开关
+    # 当type为radio单选框,checkbox复选框时,option表示可选项值列表
+    # option: Yes,No
--- a/manager/src/main/resources/sureness.yml
+++ b/manager/src/main/resources/sureness.yml
@@ -10,6 +10,7 @@ resourceRole:
 # load api resource which do not need be protected, means them need be excluded.
 # these api resource can be access by everyone
 excludedResource:
+  - /**===*
  - /account/auth/form===post
  - /**/*.html===get
  - /**/*.js===get
--- a/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts
+++ b/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.ts
@@ -102,7 +102,7 @@ export class MonitorNewComponent implements OnInit {
      "params": this.params
    };
    this.isSpinning = true;
-    this.monitorSvc.newMonitor(detectMonitor)
+    this.monitorSvc.detectMonitor(detectMonitor)
      .subscribe(message => {
        this.isSpinning = false;
        if (message.code === 0) {
--- a/web-app/src/assets/tmp/app-data.json
+++ b/web-app/src/assets/tmp/app-data.json
@@ -35,8 +35,8 @@
          "icon": "anticon-cloud",
          "children": [
            {
-              "text": "http",
-              "link": "/monitors?app=http",
+              "text": "api",
+              "link": "/monitors?app=api",
              "i18n": "monitor.app.http"
            },
            {
--- a/web-app/src/assets/tmp/i18n/zh-CN.json
+++ b/web-app/src/assets/tmp/i18n/zh-CN.json
@@ -28,6 +28,7 @@
    },
    "app": {
      "": "监控类型",
+      "api": "HTTP API",
      "http": "HTTP API",
      "ping": "PING连通性",
      "telnet": "Telnet端口可用性",