[monitor] 设计应用监控的采集定义结构yml，参数定义结构yml

common/pom.xml

@@ -13,6 +13,12 @@
 
 
     <dependencies>
+        <!-- spring -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <scope>provided</scope>
+        </dependency>
         <!-- etcd -->
         <dependency>
             <groupId>io.etcd</groupId>

common/src/main/java/com/usthe/common/entity/dto/Message.java

@@ -0,0 +1,40 @@
+package com.usthe.common.entity.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * Unified message structure definition for front and back ends
+ *
+ * {
+ *   data:{....},
+ *   msg: message,
+ *   code: 3432
+ * }
+ * @author tomsun28
+ * @date 23:48 2019/08/01
+ */
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class Message<T> {
+
+    /**
+     * message body data
+     */
+    private T data;
+
+    /**
+     * exception message when error happen or success message
+     */
+    private String msg;
+
+    /**
+     * response code, not http code
+     */
+    private Integer code;
+
+}

common/src/main/java/com/usthe/common/entity/job/Metrics.java

@@ -67,4 +67,25 @@ public class Metrics {
      * 使用公共的jdbc规范实现的数据库配置信息
      */
     private JdbcProtocol jdbc;
+
+    /**
+     * todo 替换指标信息
+     */
+    @Data
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public class Field {
+        /**
+         * 指标名称
+         */
+        private String field;
+        /**
+         * 指标类型 number:数字 string:字符串
+         */
+        private String type;
+        /**
+         * 指标单位
+         */
+        private String unit;
+    }
 }

manager/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>manager</artifactId>
 
     <properties>
-
+        <mysql.version>8.0.16</mysql.version>
     </properties>
 
     <dependencies>
@@ -38,6 +38,31 @@
             <artifactId>spring-boot-configuration-processor</artifactId>
             <optional>true</optional>
         </dependency>
+        <!-- data jdbc -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+        </dependency>
+        <!-- redis cache -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-cache</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-redis</artifactId>
+        </dependency>
+        <!-- mysql -->
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>${mysql.version}</version>
+            <!--<scope>runtime</scope>-->
+        </dependency>
         <!-- swagger -->
         <dependency>
             <groupId>io.springfox</groupId>

manager/src/main/java/com/usthe/manager/support/GlobalExceptionHandler.java

@@ -0,0 +1,95 @@
+package com.usthe.manager.support;
+
+
+import com.usthe.common.entity.dto.Message;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.dao.DataAccessException;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.HttpRequestMethodNotSupportedException;
+import org.springframework.web.bind.MethodArgumentNotValidException;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestControllerAdvice;
+
+/**
+ * controller exception handler
+ * @author tomsun28
+ * @date 22:45 2019-08-01
+ */
+@RestControllerAdvice
+@Slf4j
+public class GlobalExceptionHandler {
+
+    /**
+     * handler the exception thrown for data input verify
+     * @param exception data input verify exception
+     * @return response
+     */
+    @ExceptionHandler(MethodArgumentNotValidException.class)
+    @ResponseBody
+    ResponseEntity<Message> handleInputValidException(MethodArgumentNotValidException exception) {
+        StringBuffer errorMessage = new StringBuffer();
+        if (exception != null) {
+            exception.getBindingResult().getAllErrors().forEach(error ->
+                    errorMessage.append(error.getDefaultMessage()).append("."));
+        }
+        if (log.isDebugEnabled()) {
+            log.debug("[sample-tom]-[input argument not valid happen]-{}", errorMessage, exception);
+        }
+        Message message = Message.builder().msg(errorMessage.toString()).build();
+        return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(message);
+    }
+
+    /**
+     * handler the exception thrown for datastore error
+     * @param exception datastore exception
+     * @return response
+     */
+    @ExceptionHandler(DataAccessException.class)
+    @ResponseBody
+    ResponseEntity<Message> handleDataAccessException(DataAccessException exception) {
+        String errorMessage = "database error happen";
+        if (exception != null) {
+            errorMessage = exception.getMessage();
+        }
+        log.warn("[sample-tom]-[database error happen]-{}", errorMessage, exception);
+        Message message = Message.builder().msg(errorMessage).build();
+        return ResponseEntity.status(HttpStatus.CONFLICT).body(message);
+    }
+
+    /**
+     * handle Request method not supported
+     * @param exception Request method not supported
+     * @return response
+     */
+    @ExceptionHandler(HttpRequestMethodNotSupportedException.class)
+    @ResponseBody
+    ResponseEntity<Message> handleMethodNotSupportException(HttpRequestMethodNotSupportedException exception) {
+        String errorMessage = "Request method not supported";
+        if (exception != null && exception.getMessage() != null) {
+            errorMessage = exception.getMessage();
+        }
+        log.info("[monitor]-[Request method not supported]-{}", errorMessage);
+        Message message = Message.builder().msg(errorMessage).build();
+        return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).body(message);
+    }
+
+    /**
+     * handler the exception thrown for unCatch and unKnown
+     * @param exception UnknownException
+     * @return response
+     */
+    @ExceptionHandler(Exception.class)
+    @ResponseBody
+    ResponseEntity<Message> handleUnknownException(Exception exception) {
+        String errorMessage = "unknown error happen";
+        if (exception != null) {
+            errorMessage = exception.getMessage();
+        }
+        log.error("[monitor]-[unknown error happen]-{}", errorMessage, exception);
+        Message message = Message.builder().msg(errorMessage).build();
+        return ResponseEntity.status(HttpStatus.CONFLICT).body(message);
+    }
+
+}

manager/src/main/java/com/usthe/manager/support/SpringContextHolder.java

@@ -0,0 +1,48 @@
+package com.usthe.manager.support;
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
+
+/**
+ * Spring的ApplicationContext的持有者,可以用静态方法的方式获取spring容器中的bean
+ * @author tomsun28
+ * @date 21:07 2018/4/18
+ */
+@Component
+public class SpringContextHolder implements ApplicationContextAware {
+
+    private static ApplicationContext applicationContext;
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        set(applicationContext);
+    }
+
+    private static void set(ApplicationContext applicationContext) {
+        SpringContextHolder.applicationContext = applicationContext;
+    }
+
+    public static ApplicationContext getApplicationContext() {
+        assertApplicationContext();
+        return applicationContext;
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> T getBean(String beanName) {
+        assertApplicationContext();
+        return (T) applicationContext.getBean(beanName);
+    }
+
+    public static <T> T getBean(Class<T> tClass) {
+        assertApplicationContext();
+        return (T) applicationContext.getBean(tClass);
+    }
+
+    private static void assertApplicationContext() {
+        if (null == SpringContextHolder.applicationContext) {
+            throw new RuntimeException("applicationContext为空,请检查是否注入springContextHolder");
+        }
+    }
+}

manager/src/main/resources/db/data.sql

@@ -0,0 +1,71 @@
+use tom ;
+-- ----------------------------
+-- Records of auth_resource
+-- ----------------------------
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (101, 'User get token', 'ACCOUNT_TOKEN', '/auth/token', 'account', 'POST', 9, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (102, 'User register', 'ACCOUNT_REGISTER', '/auth/register', 'account', 'POST', 9, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (103, 'Add resource', 'ADD_RESOURCE', '/resource', 'resource', 'POST', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (104, 'Update resource', 'UPDATE_RESOURCE', '/resource', 'resource', 'PUT', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (105, 'Delete resource', 'DELETE_RESOURCE', '/resource/*', 'resource', 'DELETE', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (106, 'Get resource', 'GET_RESOURCES', '/resource/*/*', 'resource', 'GET', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (107, 'Add role', 'ADD_ROLE', '/role', 'role', 'POST', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (108, 'Update role', 'UPDATE_ROLE', '/role', 'role', 'PUT', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (109, 'Delete role', 'DELETE_ROLE', '/role/*', 'role', 'DELETE', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (110, 'Get role', 'GET_ROLES', '/role/*/*', 'role', 'GET', 1, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (111, 'User get custom token', 'ACCOUNT_CUSTOM_TOKEN', '/auth/custom/token', 'account', 'POST', 9, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (112, 'Static Resource', 'Static Resource', '/**/*.html', 'static', 'GET', 9, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (113, 'Static Resource', 'Static Resource', '/**/*.js', 'static', 'GET', 9, null);
+insert into auth_resource (id, name, code, uri, type, method, status, description) values (114, 'Static Resource', 'Static Resource', '/**/*.css', 'static', 'GET', 9, null);
+
+-- ----------------------------
+-- Records of auth_role
+-- ----------------------------
+insert into auth_role (id, name, code, status, description) values (100, 'admin role', 'role_admin', 1, null);
+insert into auth_role (id, name, code, status, description) values (102, 'user role', 'role_user', 1, null);
+insert into auth_role (id, name, code, status, description) values (103, 'guest role', 'role_guest', 1, null);
+
+-- ----------------------------
+-- Records of auth_role_resource_bind
+-- ----------------------------
+-- role_admin has these resource
+insert into auth_role_resource_bind (id, role_id, resource_id) values (1, 100, 101);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (2, 100, 102);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (3, 100, 103);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (4, 100, 104);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (5, 100, 105);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (6, 100, 106);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (7, 100, 107);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (8, 100, 108);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (9, 100, 109);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (10, 100, 110);
+
+-- role_user has "get,add,update" related resources, do not have "delete" related resources
+insert into auth_role_resource_bind (id, role_id, resource_id) values (11, 102, 103);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (12, 102, 104);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (13, 102, 106);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (14, 102, 107);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (15, 102, 108);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (16, 102, 110);
+
+-- role_guest has "get" related resources
+insert into auth_role_resource_bind (id, role_id, resource_id) values (17, 103, 106);
+insert into auth_role_resource_bind (id, role_id, resource_id) values (18, 103, 110);
+
+
+-- ----------------------------
+-- Records of auth_user
+-- ----------------------------
+insert into  auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (111, 'admin', 'admin', null, null, null, null, null, 1, 1);
+insert into  auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (112, 'user', 'user', null, null, null, null, null, 1, 1);
+insert into  auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (113, 'guest', 'guest', null, null, null, null, null, 1, 1);
+insert into  auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (114, 'noRole', 'noRole', null, null, null, null, null, 1, 1);
+
+
+
+-- ----------------------------
+-- Records of auth_user_role_bind
+-- ----------------------------
+insert into  auth_user_role_bind (id, user_id, role_id) values (12, 111, 100);
+insert into  auth_user_role_bind (id, user_id, role_id) values (13, 111, 102);
+insert into  auth_user_role_bind (id, user_id, role_id) values (14, 112, 102);
+insert into  auth_user_role_bind (id, user_id, role_id) values (15, 113, 103);

manager/src/main/resources/db/schema.sql

@@ -0,0 +1,58 @@
+use usthe;
+
+-- ----------------------------
+-- Table structure for monitor
+-- ----------------------------
+DROP TABLE IF EXISTS  monitor ;
+CREATE TABLE  monitor
+(
+     id           bigint       not null auto_increment comment '监控ID',
+     jobId        bigint       not null comment '监控对应下发的任务ID',
+     name         varchar(100) not null comment '监控的名称',
+     app          varchar(100) not null comment '监控的类型:linux,mysql,jvm...',
+     host         varchar(100) not null comment '监控的对端host:ipv4,ipv6,域名',
+     intervals    int          not null default 600 comment '监控的采集间隔时间,单位秒',
+     status       smallint(4)  not null default 1 comment '监控状态 0:未监控,1:可用,2:不可用,3:不可达,4:挂起',
+     description  varchar(255) comment '监控描述信息',
+     creator      varchar(100) comment '创建者',
+     modifier     varchar(100) comment '最新修改者',
+     gmt_create   timestamp    default current_timestamp comment 'create time',
+     gmt_update   datetime     default current_timestamp on update current_timestamp comment 'update time',
+     primary key (id)
+) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for param
+-- ----------------------------
+DROP TABLE IF EXISTS  param ;
+CREATE TABLE  param
+(
+    id           bigint       not null auto_increment comment '参数ID',
+    monitor_id   bigint       not null comment '监控ID',
+    param_name   varchar(100) not null comment '参数显示名称',
+    param_key    varchar(100) not null comment '参数标识符',
+    param_value  varchar(255) not null comment '参数值',
+    param_type   smallint(4)  not null default 0 comment '参数类型 0:数字 1:字符串 2:加密串',
+    gmt_create   timestamp    default current_timestamp comment 'create time',
+    gmt_update   datetime     default current_timestamp on update current_timestamp comment 'update time',
+    primary key (id),
+    index monitor_id (monitor_id)
+) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for param
+-- ----------------------------
+DROP TABLE IF EXISTS  param_define ;
+CREATE TABLE  param_define
+(
+    id           bigint       not null auto_increment comment '参数ID',
+    monitor_id   bigint       not null comment '监控ID',
+    param_name   varchar(100) not null comment '参数显示名称',
+    param_key    varchar(100) not null comment '参数标识符',
+    param_value  varchar(255) not null comment '参数值',
+    param_type   smallint(4)  not null default 0 comment '参数类型 0:数字 1:字符串 2:加密串',
+    gmt_create   timestamp    default current_timestamp comment 'create time',
+    gmt_update   datetime     default current_timestamp on update current_timestamp comment 'update time',
+    primary key (id),
+    index monitor_id (monitor_id)
+) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4;

manager/src/main/resources/define/collect/example.yml

@@ -0,0 +1,105 @@
+# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
+app: cloud
+# 参数映射map. type是参数类型: number数字, string明文字符串, secret加密字符串
+# 强制固定必须参数 - host
+configmap:
+  - key: host
+    type: string
+  - key: port
+    type: number
+  - key: username
+    type: string
+  - key: password
+    type: secret
+  - key: param1
+    type: string
+# 指标组列表
+metrics:
+  # 第一个监控指标组 cpu
+  - name: cpu
+    # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
+    # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
+    priority: 0
+    # 指标组中的具体监控指标
+    fields:
+      # 指标信息 包括 field名称, type字段类型:number数字,string字符串, unit:指标单位
+      - field: usage
+        type: number
+        unit: %
+      - field: cores
+        type: number
+      - field: waitime
+        type: number
+        unit: s
+# (非必须)监控指标别名，与上面的指标名映射。用于采集接口数据字段不直接是最终指标名称,需要此别名做映射转换
+    aliasFields:
+      - core1
+      - core2
+      - usage
+      - allTime
+      - runningTime
+# (非必须)指标计算表达式,与上面的别名一起作用,计算出最终需要的指标值
+# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
+    calculates:
+      - cores=core1+core2
+      - usage=usage
+      - waitTime=allTime-runningTime
+# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
+    protocol: http
+# 当protocol为http协议时具体的采集配置
+    http:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      # url请求接口路径
+      url: /cpu
+      # 请求方式 GET POST PUT DELETE PATCH
+      method: GET
+      # 是否启用ssl/tls,即是http还是https,默认false
+      ssl: false
+      # 请求头内容
+      headers:
+        apiVersion: 'v1'
+      # 请求参数内容
+      params:
+        - param1: param1
+        - param2: param2
+      # 认证
+      authorization:
+        # 认证方式: Basic Auth, Digest Auth, Bearer Token
+        type: Basic Auth
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+      # 响应数据解析方式: default-系统规则,json_path-jsonPath脚本,xml_path-xmlPath脚本,prometheus-Prometheus数据规则
+      parseType: jsonPath
+      parseScript: "$.cpu[:1].*"
+
+  - name: memory
+    priority: 1
+    fields:
+      - field: total
+        type: number
+        unit: kb
+      - field: usage
+        type: number
+        unit: %
+      - field: speed
+        type: number
+    protocol: http
+    http:
+      host: ^_^host^_^
+      port: ^_^port^_^
+      url: /memory
+      method: GET
+      headers:
+        apiVersion: 'v1'
+      params:
+        - param1: param1
+        - param2: param2
+      authorization:
+        type: Basic Auth
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+      parseType: jsonPath
+      parseScript: "$.memory[:1].*"

manager/src/main/resources/define/param/example.yml

@@ -0,0 +1,23 @@
+# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
+app: cloud
+# 强制固定必须参数 - host(ipv4,ipv6,域名)
+param:
+  # field-字段名称标识符 type-字段类型,样式(大部分映射input标签type属性)
+  - field: host
+    type: host
+  - field: port
+    type: number
+    # 当type为number时,用range表示范围
+    range: 0-255
+  - field: username
+    type: text
+    # 当type为text时,用limit表示字符串限制大小
+    limit: 20
+  - field: password
+    type: password
+  - field: param1
+    type: radio
+    # 当type为radio单选框,checkbox复选框时,option表示可选项值列表
+    option:
+      - param3
+      - param4