[collector,manager]feature 支持microsoft sqlserver数据库监控类型 (#37)

2022-03-18 10:19:17 +08:00
parent a6a5f6abb4
commit 1f52ae4a88
8 changed files with 212 additions and 8 deletions
--- a/collector/pom.xml
+++ b/collector/pom.xml
@@ -103,6 +103,12 @@
            <artifactId>sshd-core</artifactId>
            <version>2.8.0</version>
        </dependency>
+        <!-- sql server -->
+        <dependency>
+            <groupId>com.microsoft.sqlserver</groupId>
+            <artifactId>mssql-jdbc</artifactId>
+            <version>10.2.0.jre8</version>
+        </dependency>
    </dependencies>

 </project>
--- a/collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java
+++ b/collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java
@@ -56,7 +56,9 @@ public class JdbcCommonCollect extends AbstractCollect {
        int timeout = 3000;
        try {
            // 获取查询语句超时时间
-            timeout = Integer.parseInt(jdbcProtocol.getTimeout());
+            if (jdbcProtocol.getTimeout() != null) {
+                timeout = Integer.parseInt(jdbcProtocol.getTimeout());
+            }
        } catch (Exception e) {
            log.warn(e.getMessage());
        }
@@ -140,7 +142,9 @@ public class JdbcCommonCollect extends AbstractCollect {
        Connection connection = DriverManager.getConnection(url, username, password);
        statement = connection.createStatement();
        // 设置查询超时时间10秒
-        statement.setQueryTimeout(timeout);
+        int timeoutSecond = timeout / 1000;
+        timeoutSecond = timeoutSecond <= 0 ? 1 : timeoutSecond;
+        statement.setQueryTimeout(timeoutSecond);
        // 设置查询最大行数1000行
        statement.setMaxRows(1000);
        JdbcConnect jdbcConnect = new JdbcConnect(connection);
@@ -200,7 +204,7 @@ public class JdbcCommonCollect extends AbstractCollect {
            HashMap<String, String> values = new HashMap<>(columns.size());
            while (resultSet.next()) {
                if (resultSet.getString(1) != null) {
-                    values.put(resultSet.getString(1).toLowerCase(), resultSet.getString(2));
+                    values.put(resultSet.getString(1).toLowerCase().trim(), resultSet.getString(2));
                }
            }
            CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
@@ -277,6 +281,10 @@ public class JdbcCommonCollect extends AbstractCollect {
                url = "jdbc:postgresql://" + jdbcProtocol.getHost() + ":" + jdbcProtocol.getPort()
                        + "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase());
                break;
+            case "sqlserver":
+                url = "jdbc:sqlserver://" + jdbcProtocol.getHost() + ":" + jdbcProtocol.getPort()
+                        + ";" + (jdbcProtocol.getDatabase() == null ? "" : "DatabaseName=" + jdbcProtocol.getDatabase());
+                break;
            default:
                throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform());

--- a/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java
+++ b/collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;

 /**
@@ -172,14 +173,23 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
        if (metrics.getCalculates() == null) {
            metrics.setCalculates(Collections.emptyList());
        }
+        // eg: database_pages=Database pages 非常规映射
+        Map<String, String> fieldAliasMap = new HashMap<>(8);
        Map<String, Expression> fieldExpressionMap = metrics.getCalculates()
                .stream()
                .map(cal -> {
                    int splitIndex = cal.indexOf("=");
                    String field = cal.substring(0, splitIndex);
                    String expressionStr = cal.substring(splitIndex + 1);
-                    Expression expression = AviatorEvaluator.compile(expressionStr, true);
+                    Expression expression = null;
+                    try {
+                        expression = AviatorEvaluator.compile(expressionStr, true);
+                    } catch (Exception e) {
+                        fieldAliasMap.put(field, expressionStr);
+                        return null;
+                    }
                    return new Object[]{field, expression}; })
+                .filter(Objects::nonNull)
                .collect(Collectors.toMap(arr -> (String)arr[0], arr -> (Expression) arr[1]));

        List<Metrics.Field> fields = metrics.getFields();
@@ -226,7 +236,12 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
                    }
                } else {
                    // 不存在 则映射别名值
-                    value = aliasFieldValueMap.get(realField);
+                    String aliasField = fieldAliasMap.get(realField);
+                    if (aliasField != null) {
+                        value = aliasFieldValueMap.get(aliasField);
+                    } else {
+                        value = aliasFieldValueMap.get(realField);
+                    }
                }
                if (value == null) {
                    value = CommonConstants.NULL_VALUE;
--- a/manager/src/main/resources/define/app/sqlserver.yml
+++ b/manager/src/main/resources/define/app/sqlserver.yml
@@ -0,0 +1,136 @@
+category: db
+app: sqlserver
+name:
+  zh-CN: SqlServer数据库
+  en-US: SqlServer DB
+# 参数映射map. type是参数类型: 0-number数字, 1-string明文字符串, 2-secret加密字符串
+# 强制固定必须参数 - host
+configmap:
+  - key: host
+    type: 1
+  - key: port
+    type: 0
+  - key: username
+    type: 1
+  - key: password
+    type: 2
+  - key: database
+    type: 1
+  - key: timeout
+    type: 0
+  - key: url
+    type: 1
+# 指标组列表
+metrics:
+  - name: basic
+    # 指标组调度优先级(0-127)越小优先级越高,优先级低的指标组会等优先级高的指标组采集完成后才会被调度,相同优先级的指标组会并行调度采集
+    # 优先级为0的指标组为可用性指标组,即它会被首先调度,采集成功才会继续调度其它指标组,采集失败则中断调度
+    priority: 0
+    # 指标组中的具体监控指标
+    fields:
+      # 指标信息 包括 field名称   type字段类型:0-number数字,1-string字符串   instance是否为实例主键   unit:指标单位
+      - field: machine_name
+        type: 1
+        instance: true
+      - field: server_name
+        type: 1
+      - field: version
+        type: 1
+      - field: edition
+        type: 1
+      - field: start_time
+        type: 1
+    protocol: jdbc
+    jdbc:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      platform: sqlserver
+      username: ^_^username^_^
+      password: ^_^password^_^
+      database: ^_^database^_^
+      timeout: ^_^timeout^_^
+      # SQL查询方式： oneRow, multiRow, columns
+      queryType: oneRow
+      # sql
+      sql: SELECT SERVERPROPERTY('MachineName') AS machine_name, SERVERPROPERTY('ServerName') AS server_name, SERVERPROPERTY('ProductVersion') AS version, SERVERPROPERTY('Edition') AS edition, sqlserver_start_time AS start_time FROM sys.dm_os_sys_info;
+      url: ^_^url^_^
+
+  - name: performance_counters
+    priority: 1
+    fields:
+      # 指标信息 包括 field名称   type字段类型:0-number数字,1-string字符串   instance是否为实例主键   unit:指标单位
+      - field: database_pages
+        type: 0
+      - field: target_pages
+        type: 0
+      - field: page_life_expectancy
+        type: 0
+      - field: buffer_cache_hit_ratio
+        type: 0
+      - field: checkpoint_pages_sec
+        type: 0
+      - field: page_reads_sec
+        type: 0
+      - field: page_writes_sec
+        type: 0
+    # (非必须)监控指标别名，与上面的指标名映射。用于采集接口数据字段不直接是最终指标名称,需要此别名做映射转换
+    aliasFields:
+      - Database pages
+      - Target pages
+      - Page life expectancy
+      - Buffer cache hit ratio
+      - Checkpoint pages/sec
+      - Page reads/sec
+      - Page writes/sec
+    # (非必须)指标计算表达式,与上面的别名一起作用,计算出最终需要的指标值
+    # eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
+    calculates:
+      - database_pages=Database pages
+      - target_pages=Target pages
+      - page_life_expectancy=Page life expectancy
+      - buffer_cache_hit_ratio=Buffer cache hit ratio
+      - checkpoint_pages_sec=Checkpoint pages/sec
+      - page_reads_sec=Page reads/sec
+      - page_writes_sec=Page writes/sec
+    protocol: jdbc
+    jdbc:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      platform: sqlserver
+      username: ^_^username^_^
+      password: ^_^password^_^
+      database: ^_^database^_^
+      timeout: ^_^timeout^_^
+      # SQL查询方式： oneRow, multiRow, columns
+      queryType: columns
+      # sql
+      sql: select counter_name, cntr_value from sys.dm_os_performance_counters where object_name = 'SQLServer:Buffer Manager';
+      url: ^_^url^_^
+
+  - name: connection
+    priority: 1
+    fields:
+      # 指标信息 包括 field名称   type字段类型:0-number数字,1-string字符串   instance是否为实例主键   unit:指标单位
+      - field: connection
+        type: 0
+        unit: 连接数
+    protocol: jdbc
+    jdbc:
+      # 主机host: ipv4 ipv6 域名
+      host: ^_^host^_^
+      # 端口
+      port: ^_^port^_^
+      platform: sqlserver
+      username: ^_^username^_^
+      password: ^_^password^_^
+      database: ^_^database^_^
+      timeout: ^_^timeout^_^
+      # SQL查询方式： oneRow, multiRow, columns
+      queryType: oneRow
+      # sql
+      sql: SELECT cntr_value as connection FROM sys.dm_os_performance_counters WHERE object_name = 'SQLServer:General Statistics' AND counter_name = 'User Connections';
+      url: ^_^url^_^
--- a/manager/src/main/resources/define/param/mariadb.yml
+++ b/manager/src/main/resources/define/param/mariadb.yml
@@ -9,7 +9,7 @@ param:
    type: number
    range: '[0,65535]'
    required: true
-    defaultValue: 80
+    defaultValue: 3306
    placeholder: '请输入端口'
  - field: timeout
    name: 查询超时时间
--- a/manager/src/main/resources/define/param/mysql.yml
+++ b/manager/src/main/resources/define/param/mysql.yml
@@ -9,7 +9,7 @@ param:
    type: number
    range: '[0,65535]'
    required: true
-    defaultValue: 80
+    defaultValue: 3306
    placeholder: '请输入端口'
  - field: timeout
    name: 查询超时时间
--- a/manager/src/main/resources/define/param/sqlserver.yml
+++ b/manager/src/main/resources/define/param/sqlserver.yml
@@ -0,0 +1,36 @@
+app: sqlserver
+param:
+  - field: host
+    name: 主机Host
+    type: host
+    required: true
+  - field: port
+    name: 端口
+    type: number
+    range: '[0,65535]'
+    required: true
+    defaultValue: 1433
+    placeholder: '请输入端口'
+  - field: timeout
+    name: 查询超时时间
+    type: number
+    required: false
+    defaultValue: 3000
+    placeholder: '查询超时时间'
+  - field: database
+    name: 数据库名称
+    type: text
+    required: false
+  - field: username
+    name: 用户名
+    type: text
+    limit: 20
+    required: false
+  - field: password
+    name: 密码
+    type: password
+    required: false
+  - field: url
+    name: URL
+    type: text
+    required: false
--- a/warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java
+++ b/warehouse/src/main/java/com/usthe/warehouse/store/TdEngineDataStorage.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;

 /**
 * influxdb存储采集数据
@@ -39,6 +40,7 @@ public class TdEngineDataStorage implements DisposableBean {
    private HikariDataSource hikariDataSource;
    private WarehouseWorkerPool workerPool;
    private MetricsDataExporter dataExporter;
+    private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')");
    private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO %s USING %s TAGS (%s) VALUES %s";
    private static final String CREATE_SUPER_TABLE_SQL = "CREATE STABLE IF NOT EXISTS %s %s TAGS (monitor BIGINT)";
    private static final String NO_SUPER_TABLE_ERROR = "Table does not exist";
@@ -200,8 +202,9 @@ public class TdEngineDataStorage implements DisposableBean {
    }

    private String formatStringValue(String value){
-        return value.replaceAll("(\\\\)|(')","\\\\$0");
+        return SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
    }
+
    @Override
    public void destroy() throws Exception {
        if (hikariDataSource != null) {