OpenSource/hertzbeat
Public Access
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[monitor] 设计应用监控的采集定义结构yml，参数定义结构yml

Browse Source
This commit is contained in:
tomsun28
2021-11-13 21:44:03 +08:00
parent b1ff63883d
commit 0760a6d5f9
11 changed files with 493 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java
View File

@@ -31,7 +31,6 @@ import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**

6
common/pom.xml
View File

@@ -13,6 +13,12 @@
<dependencies>
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<scope>provided</scope>
</dependency>
<!-- etcd -->
<dependency>
<groupId>io.etcd</groupId>

40
common/src/main/java/com/usthe/common/entity/dto/Message.java Normal file
View File

@@ -0,0 +1,40 @@
package com.usthe.common.entity.dto;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Unified message structure definition for front and back ends
*
* {
* data:{....},
* msg: message,
* code: 3432
* }
* @author tomsun28
* @date 23:48 2019/08/01
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Message<T> {
/**
* message body data
*/
private T data;
/**
* exception message when error happen or success message
*/
private String msg;
/**
* response code, not http code
*/
private Integer code;
}

21
common/src/main/java/com/usthe/common/entity/job/Metrics.java
View File

@@ -67,4 +67,25 @@ public class Metrics {
* 使用公共的jdbc规范实现的数据库配置信息
*/
private JdbcProtocol jdbc;
/**
* todo 替换指标信息
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Field {
/**
* 指标名称
*/
private String field;
/**
* 指标类型 number:数字 string:字符串
*/
private String type;
/**
* 指标单位
*/
private String unit;
}
}

27
manager/pom.xml
View File

@@ -12,7 +12,7 @@
<artifactId>manager</artifactId>
<properties>
<mysql.version>8.0.16</mysql.version>
</properties>
<dependencies>
@@ -38,6 +38,31 @@
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<!-- data jdbc -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<!-- redis cache -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<!-- mysql -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.version}</version>
<!--<scope>runtime</scope>-->
</dependency>
<!-- swagger -->
<dependency>
<groupId>io.springfox</groupId>

95
manager/src/main/java/com/usthe/manager/support/GlobalExceptionHandler.java Normal file
View File

@@ -0,0 +1,95 @@
package com.usthe.manager.support;
import com.usthe.common.entity.dto.Message;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DataAccessException;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.HttpRequestMethodNotSupportedException;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestControllerAdvice;
/**
* controller exception handler
* @author tomsun28
* @date 22:45 2019-08-01
*/
@RestControllerAdvice
@Slf4j
public class GlobalExceptionHandler {
/**
* handler the exception thrown for data input verify
* @param exception data input verify exception
* @return response
*/
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseBody
ResponseEntity<Message> handleInputValidException(MethodArgumentNotValidException exception) {
StringBuffer errorMessage = new StringBuffer();
if (exception != null) {
exception.getBindingResult().getAllErrors().forEach(error ->
errorMessage.append(error.getDefaultMessage()).append("."));
}
if (log.isDebugEnabled()) {
log.debug("[sample-tom]-[input argument not valid happen]-{}", errorMessage, exception);
}
Message message = Message.builder().msg(errorMessage.toString()).build();
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(message);
}
/**
* handler the exception thrown for datastore error
* @param exception datastore exception
* @return response
*/
@ExceptionHandler(DataAccessException.class)
@ResponseBody
ResponseEntity<Message> handleDataAccessException(DataAccessException exception) {
String errorMessage = "database error happen";
if (exception != null) {
errorMessage = exception.getMessage();
}
log.warn("[sample-tom]-[database error happen]-{}", errorMessage, exception);
Message message = Message.builder().msg(errorMessage).build();
return ResponseEntity.status(HttpStatus.CONFLICT).body(message);
}
/**
* handle Request method not supported
* @param exception Request method not supported
* @return response
*/
@ExceptionHandler(HttpRequestMethodNotSupportedException.class)
@ResponseBody
ResponseEntity<Message> handleMethodNotSupportException(HttpRequestMethodNotSupportedException exception) {
String errorMessage = "Request method not supported";
if (exception != null && exception.getMessage() != null) {
errorMessage = exception.getMessage();
}
log.info("[monitor]-[Request method not supported]-{}", errorMessage);
Message message = Message.builder().msg(errorMessage).build();
return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).body(message);
}
/**
* handler the exception thrown for unCatch and unKnown
* @param exception UnknownException
* @return response
*/
@ExceptionHandler(Exception.class)
@ResponseBody
ResponseEntity<Message> handleUnknownException(Exception exception) {
String errorMessage = "unknown error happen";
if (exception != null) {
errorMessage = exception.getMessage();
}
log.error("[monitor]-[unknown error happen]-{}", errorMessage, exception);
Message message = Message.builder().msg(errorMessage).build();
return ResponseEntity.status(HttpStatus.CONFLICT).body(message);
}
}

48
manager/src/main/java/com/usthe/manager/support/SpringContextHolder.java Normal file
View File

@@ -0,0 +1,48 @@
package com.usthe.manager.support;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
/**
* Spring的ApplicationContext的持有者,可以用静态方法的方式获取spring容器中的bean
* @author tomsun28
* @date 21:07 2018/4/18
*/
@Component
public class SpringContextHolder implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
set(applicationContext);
}
private static void set(ApplicationContext applicationContext) {
SpringContextHolder.applicationContext = applicationContext;
}
public static ApplicationContext getApplicationContext() {
assertApplicationContext();
return applicationContext;
}
@SuppressWarnings("unchecked")
public static <T> T getBean(String beanName) {
assertApplicationContext();
return (T) applicationContext.getBean(beanName);
}
public static <T> T getBean(Class<T> tClass) {
assertApplicationContext();
return (T) applicationContext.getBean(tClass);
}
private static void assertApplicationContext() {
if (null == SpringContextHolder.applicationContext) {
throw new RuntimeException("applicationContext为空,请检查是否注入springContextHolder");
}
}
}

71
manager/src/main/resources/db/data.sql Normal file
View File

@@ -0,0 +1,71 @@
use tom ;
-- ----------------------------
-- Records of auth_resource
-- ----------------------------
insert into auth_resource (id, name, code, uri, type, method, status, description) values (101, 'User get token', 'ACCOUNT_TOKEN', '/auth/token', 'account', 'POST', 9, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (102, 'User register', 'ACCOUNT_REGISTER', '/auth/register', 'account', 'POST', 9, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (103, 'Add resource', 'ADD_RESOURCE', '/resource', 'resource', 'POST', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (104, 'Update resource', 'UPDATE_RESOURCE', '/resource', 'resource', 'PUT', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (105, 'Delete resource', 'DELETE_RESOURCE', '/resource/*', 'resource', 'DELETE', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (106, 'Get resource', 'GET_RESOURCES', '/resource/*/*', 'resource', 'GET', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (107, 'Add role', 'ADD_ROLE', '/role', 'role', 'POST', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (108, 'Update role', 'UPDATE_ROLE', '/role', 'role', 'PUT', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (109, 'Delete role', 'DELETE_ROLE', '/role/*', 'role', 'DELETE', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (110, 'Get role', 'GET_ROLES', '/role/*/*', 'role', 'GET', 1, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (111, 'User get custom token', 'ACCOUNT_CUSTOM_TOKEN', '/auth/custom/token', 'account', 'POST', 9, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (112, 'Static Resource', 'Static Resource', '/**/*.html', 'static', 'GET', 9, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (113, 'Static Resource', 'Static Resource', '/**/*.js', 'static', 'GET', 9, null);
insert into auth_resource (id, name, code, uri, type, method, status, description) values (114, 'Static Resource', 'Static Resource', '/**/*.css', 'static', 'GET', 9, null);
-- ----------------------------
-- Records of auth_role
-- ----------------------------
insert into auth_role (id, name, code, status, description) values (100, 'admin role', 'role_admin', 1, null);
insert into auth_role (id, name, code, status, description) values (102, 'user role', 'role_user', 1, null);
insert into auth_role (id, name, code, status, description) values (103, 'guest role', 'role_guest', 1, null);
-- ----------------------------
-- Records of auth_role_resource_bind
-- ----------------------------
-- role_admin has these resource
insert into auth_role_resource_bind (id, role_id, resource_id) values (1, 100, 101);
insert into auth_role_resource_bind (id, role_id, resource_id) values (2, 100, 102);
insert into auth_role_resource_bind (id, role_id, resource_id) values (3, 100, 103);
insert into auth_role_resource_bind (id, role_id, resource_id) values (4, 100, 104);
insert into auth_role_resource_bind (id, role_id, resource_id) values (5, 100, 105);
insert into auth_role_resource_bind (id, role_id, resource_id) values (6, 100, 106);
insert into auth_role_resource_bind (id, role_id, resource_id) values (7, 100, 107);
insert into auth_role_resource_bind (id, role_id, resource_id) values (8, 100, 108);
insert into auth_role_resource_bind (id, role_id, resource_id) values (9, 100, 109);
insert into auth_role_resource_bind (id, role_id, resource_id) values (10, 100, 110);
-- role_user has "get,add,update" related resources, do not have "delete" related resources
insert into auth_role_resource_bind (id, role_id, resource_id) values (11, 102, 103);
insert into auth_role_resource_bind (id, role_id, resource_id) values (12, 102, 104);
insert into auth_role_resource_bind (id, role_id, resource_id) values (13, 102, 106);
insert into auth_role_resource_bind (id, role_id, resource_id) values (14, 102, 107);
insert into auth_role_resource_bind (id, role_id, resource_id) values (15, 102, 108);
insert into auth_role_resource_bind (id, role_id, resource_id) values (16, 102, 110);
-- role_guest has "get" related resources
insert into auth_role_resource_bind (id, role_id, resource_id) values (17, 103, 106);
insert into auth_role_resource_bind (id, role_id, resource_id) values (18, 103, 110);
-- ----------------------------
-- Records of auth_user
-- ----------------------------
insert into auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (111, 'admin', 'admin', null, null, null, null, null, 1, 1);
insert into auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (112, 'user', 'user', null, null, null, null, null, 1, 1);
insert into auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (113, 'guest', 'guest', null, null, null, null, null, 1, 1);
insert into auth_user (id, username, password, salt, avatar, phone, email, sex, status, create_where) values (114, 'noRole', 'noRole', null, null, null, null, null, 1, 1);
-- ----------------------------
-- Records of auth_user_role_bind
-- ----------------------------
insert into auth_user_role_bind (id, user_id, role_id) values (12, 111, 100);
insert into auth_user_role_bind (id, user_id, role_id) values (13, 111, 102);
insert into auth_user_role_bind (id, user_id, role_id) values (14, 112, 102);
insert into auth_user_role_bind (id, user_id, role_id) values (15, 113, 103);

58
manager/src/main/resources/db/schema.sql Normal file
View File

@@ -0,0 +1,58 @@
use usthe;
-- ----------------------------
-- Table structure for monitor
-- ----------------------------
DROP TABLE IF EXISTS monitor ;
CREATE TABLE monitor
(
id bigint not null auto_increment comment '监控ID',
jobId bigint not null comment '监控对应下发的任务ID',
name varchar(100) not null comment '监控的名称',
app varchar(100) not null comment '监控的类型:linux,mysql,jvm...',
host varchar(100) not null comment '监控的对端host:ipv4,ipv6,域名',
intervals int not null default 600 comment '监控的采集间隔时间,单位秒',
status smallint(4) not null default 1 comment '监控状态 0:未监控,1:可用,2:不可用,3:不可达,4:挂起',
description varchar(255) comment '监控描述信息',
creator varchar(100) comment '创建者',
modifier varchar(100) comment '最新修改者',
gmt_create timestamp default current_timestamp comment 'create time',
gmt_update datetime default current_timestamp on update current_timestamp comment 'update time',
primary key (id)
) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Table structure for param
-- ----------------------------
DROP TABLE IF EXISTS param ;
CREATE TABLE param
(
id bigint not null auto_increment comment '参数ID',
monitor_id bigint not null comment '监控ID',
param_name varchar(100) not null comment '参数显示名称',
param_key varchar(100) not null comment '参数标识符',
param_value varchar(255) not null comment '参数值',
param_type smallint(4) not null default 0 comment '参数类型 0:数字 1:字符串 2:加密串',
gmt_create timestamp default current_timestamp comment 'create time',
gmt_update datetime default current_timestamp on update current_timestamp comment 'update time',
primary key (id),
index monitor_id (monitor_id)
) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Table structure for param
-- ----------------------------
DROP TABLE IF EXISTS param_define ;
CREATE TABLE param_define
(
id bigint not null auto_increment comment '参数ID',
monitor_id bigint not null comment '监控ID',
param_name varchar(100) not null comment '参数显示名称',
param_key varchar(100) not null comment '参数标识符',
param_value varchar(255) not null comment '参数值',
param_type smallint(4) not null default 0 comment '参数类型 0:数字 1:字符串 2:加密串',
gmt_create timestamp default current_timestamp comment 'create time',
gmt_update datetime default current_timestamp on update current_timestamp comment 'update time',
primary key (id),
index monitor_id (monitor_id)
) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4;

105
manager/src/main/resources/define/collect/example.yml Normal file
View File

@@ -0,0 +1,105 @@
# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
app: cloud
# 参数映射map. type是参数类型: number数字, string明文字符串, secret加密字符串
# 强制固定必须参数 - host
configmap:
- key: host
type: string
- key: port
type: number
- key: username
type: string
- key: password
type: secret
- key: param1
type: string
# 指标组列表
metrics:
# 第一个监控指标组 cpu
- name: cpu
# 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
# 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
priority: 0
# 指标组中的具体监控指标
fields:
# 指标信息 包括 field名称, type字段类型:number数字,string字符串, unit:指标单位
- field: usage
type: number
unit: %
- field: cores
type: number
- field: waitime
type: number
unit: s
# (非必须)监控指标别名，与上面的指标名映射。用于采集接口数据字段不直接是最终指标名称,需要此别名做映射转换
aliasFields:
- core1
- core2
- usage
- allTime
- runningTime
# (非必须)指标计算表达式,与上面的别名一起作用,计算出最终需要的指标值
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
calculates:
- cores=core1+core2
- usage=usage
- waitTime=allTime-runningTime
# 监控采集使用协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# 当protocol为http协议时具体的采集配置
http:
# 主机host: ipv4 ipv6 域名
host: ^_^host^_^
# 端口
port: ^_^port^_^
# url请求接口路径
url: /cpu
# 请求方式 GET POST PUT DELETE PATCH
method: GET
# 是否启用ssl/tls,即是http还是https,默认false
ssl: false
# 请求头内容
headers:
apiVersion: 'v1'
# 请求参数内容
params:
- param1: param1
- param2: param2
# 认证
authorization:
# 认证方式: Basic Auth, Digest Auth, Bearer Token
type: Basic Auth
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
# 响应数据解析方式: default-系统规则,json_path-jsonPath脚本,xml_path-xmlPath脚本,prometheus-Prometheus数据规则
parseType: jsonPath
parseScript: "$.cpu[:1].*"
- name: memory
priority: 1
fields:
- field: total
type: number
unit: kb
- field: usage
type: number
unit: %
- field: speed
type: number
protocol: http
http:
host: ^_^host^_^
port: ^_^port^_^
url: /memory
method: GET
headers:
apiVersion: 'v1'
params:
- param1: param1
- param2: param2
authorization:
type: Basic Auth
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
parseType: jsonPath
parseScript: "$.memory[:1].*"

23
manager/src/main/resources/define/param/example.yml Normal file
View File

@@ -0,0 +1,23 @@
# 监控应用类型名称(与文件名保持一致) eg: linux windows tomcat mysql aws...
app: cloud
# 强制固定必须参数 - host(ipv4,ipv6,域名)
param:
# field-字段名称标识符 type-字段类型,样式(大部分映射input标签type属性)
- field: host
type: host
- field: port
type: number
# 当type为number时,用range表示范围
range: 0-255
- field: username
type: text
# 当type为text时,用limit表示字符串限制大小
limit: 20
- field: password
type: password
- field: param1
type: radio
# 当type为radio单选框,checkbox复选框时,option表示可选项值列表
option:
- param3
- param4