OpenSource/hertzbeat
Public Access
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[monitor]优化去除Kafka etcd依赖

Browse Source
This commit is contained in:
tomsun28
2022-01-29 17:11:22 +08:00
parent ff38330f31
commit def62ce0a9
81 changed files with 303 additions and 1175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
collector/README.md
View File

@@ -31,18 +31,6 @@
* Ping
* 服务端口
#### HELP
1. ARK插件类隔离未生效
> 注意需构建在jdk1.8环境中运行
> 插件是否配置导入并配置正确
> 本地DEBUG时需单独IDEA打开运行collector工程，不能将plugin和collector在同一工程打开运行
2. metaspace元空间内存占用多或溢出
> 建议调整JVM参数 ```-Dsun.reflect.inflationThreshold=100000```
> 由于使用太多反射，超过参数`inflationThreshold`默认值15阈值导致触发JVM反射优化(加快反射速度),
> 反射获取类信息由使用*JNI存取器**膨胀(Inflation)*
> 为*反射每个方法生成一个类加载器DelegatingClassLoader和Java类MethodAccessor*.
> 动态加载的字节码导致PermGen持续增长.
#### HELP

19
collector/plugins/pom.xml
View File

@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>collector</artifactId>
<groupId>com.usthe.tancloud</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>plugins</artifactId>
<packaging>pom</packaging>
<modules>
<module>sample-plugin</module>
</modules>
</project>

48
collector/plugins/sample-plugin/pom.xml
View File

@@ -1,48 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>plugins</artifactId>
<groupId>com.usthe.tancloud</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>sample-plugin</artifactId>
<build>
<plugins>
<plugin>
<!--link https://www.sofastack.tech/projects/sofa-boot/sofa-ark-ark-plugin/ -->
<groupId>com.alipay.sofa</groupId>
<artifactId>sofa-ark-plugin-maven-plugin</artifactId>
<version>1.1.6</version>
<executions>
<execution>
<id>default-cli</id>
<goals>
<goal>ark-plugin</goal>
</goals>
<configuration>
<!-- 配置优先级，数字越小，优先级越高，优先启动，优先导出类，默认1000 -->
<priority>2000</priority>
<!-- 配置导出类、资源 -->
<exported>
<!-- 配置类级别导出类 -->
<classes>
<class>com.com.usthe.plugin.sample.ExportDemo</class>
</classes>
</exported>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

12
collector/plugins/sample-plugin/src/main/java/com/usthe/collector/plugin/SameClass.java
View File

@@ -1,12 +0,0 @@
package com.usthe.collector.plugin;
/**
* @author tomsun28
* @date 2021/10/8 15:12
*/
public class SameClass {
public static String hello() {
return "hello plugin";
}
}

14
collector/plugins/sample-plugin/src/main/java/com/usthe/plugin/sample/ExportDemo.java
View File

@@ -1,14 +0,0 @@
package com.usthe.plugin.sample;
import com.usthe.collector.plugin.SameClass;
/**
* @author tomsun28
* @date 2021/10/8 15:11
*/
public class ExportDemo {
public String hello() {
return SameClass.hello();
}
}

83
collector/pom.xml
View File

@@ -5,14 +5,85 @@
<parent>
<artifactId>monitor</artifactId>
<groupId>com.usthe.tancloud</groupId>
<version>1.0-SNAPSHOT</version>
<version>1.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>collector</artifactId>
<packaging>pom</packaging>
<modules>
<module>server</module>
<module>plugins</module>
</modules>
<dependencies>
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- common -->
<dependency>
<groupId>com.usthe.tancloud</groupId>
<artifactId>common</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<!-- etcd -->
<dependency>
<groupId>io.etcd</groupId>
<artifactId>jetcd-core</artifactId>
<version>0.5.11</version>
</dependency>
<!-- kafka -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>3.0.0</version>
</dependency>
<!-- http -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
<!--network-->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.8.0</version>
</dependency>
<!--json path parser-->
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.6.0</version>
</dependency>
<!-- lru hashmap -->
<dependency>
<groupId>com.googlecode.concurrentlinkedhashmap</groupId>
<artifactId>concurrentlinkedhashmap-lru</artifactId>
<version>1.4.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.0.1-jre</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
<version>5.2.7</version>
</dependency>
</dependencies>
</project>

146
collector/server/pom.xml
View File

@@ -1,146 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>collector</artifactId>
<groupId>com.usthe.tancloud</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>server</artifactId>
<dependencies>
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<!-- isolation -->
<dependency>
<groupId>com.alipay.sofa</groupId>
<artifactId>sofa-ark-springboot-starter</artifactId>
<version>1.1.6</version>
</dependency>
<!-- common -->
<dependency>
<groupId>com.usthe.tancloud</groupId>
<artifactId>common</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<!-- etcd -->
<dependency>
<groupId>io.etcd</groupId>
<artifactId>jetcd-core</artifactId>
<version>0.5.11</version>
</dependency>
<!-- kafka -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>3.0.0</version>
</dependency>
<!-- http -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
<!--network-->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.8.0</version>
</dependency>
<!--json path parser-->
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.6.0</version>
</dependency>
<!-- lru hashmap -->
<dependency>
<groupId>com.googlecode.concurrentlinkedhashmap</groupId>
<artifactId>concurrentlinkedhashmap-lru</artifactId>
<version>1.4.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.0.1-jre</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
<version>5.2.7</version>
</dependency>
<!-- plugins -->
<dependency>
<groupId>com.usthe.tancloud</groupId>
<artifactId>sample-plugin</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<finalName>hertz-beat-collector</finalName>
<plugins>
<plugin>
<groupId>com.alipay.sofa</groupId>
<artifactId>sofa-ark-maven-plugin</artifactId>
<version>1.1.6</version>
<executions>
<execution>
<id>default-cli</id>
<!--goal executed to generate executable-ark-jar -->
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<!--specify destination where executable-ark-jar will be saved, default saved to ${project.build.directory}-->
<outputDirectory>./target</outputDirectory>
<!--default none-->
<arkClassifier>executable</arkClassifier>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<executions>
<execution>
<id>make-zip</id>
<!--绑定的maven操作-->
<phase>package</phase>
<!--运行一次-->
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>../assembly/collector/assembly.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

19
collector/server/src/main/java/com/usthe/collector/Collector.java
View File

@@ -1,19 +0,0 @@
package com.usthe.collector;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;
/**
* collector start
* @author tomsun28
* @date 2021/10/7 18:02
*/
@SpringBootApplication(exclude = {KafkaAutoConfiguration.class})
@Slf4j
public class Collector {
public static void main(String[] args) {
SpringApplication.run(Collector.class, args);
}
}

23
collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConfiguration.java
View File

@@ -1,23 +0,0 @@
package com.usthe.collector.dispatch;
import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.Options;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author tomsun28
* @date 2021/11/3 12:55
*/
@Configuration
public class DispatchConfiguration {
private static final int AVIATOR_LRU_CACHE_SIZE = 1024;
@Bean
public void configAviatorEvaluator() {
// 配置AviatorEvaluator使用LRU缓存编译后的表达式
AviatorEvaluator.getInstance()
.useLRUExpressionCache(AVIATOR_LRU_CACHE_SIZE);
}
}

57
collector/server/src/main/java/com/usthe/collector/dispatch/entrance/http/CollectJobController.java
View File

@@ -1,57 +0,0 @@
package com.usthe.collector.dispatch.entrance.http;
import com.usthe.collector.dispatch.timer.TimerDispatch;
import com.usthe.common.entity.job.Job;
import com.usthe.common.entity.message.CollectRep;
import com.usthe.common.util.ProtoJsonUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.List;
/**
* 采集job管理提供api接口
* @author tomsun28
* @date 2021/11/6 13:58
*/
@RestController
public class CollectJobController {
@Autowired
private TimerDispatch timerDispatch;
/**
* 执行一次性采集任务,获取采集数据响应
* @return 采集结果
*/
@PostMapping(path = "/job/sync", consumes = MediaType.APPLICATION_JSON_VALUE,
produces = MediaType.APPLICATION_JSON_VALUE)
public Mono<List<String>> collectSyncJobData(@RequestBody Job job) {
return Mono.create(sink -> {
CollectResponseEventListener listener = new CollectResponseEventListener() {
@Override
public void response(List<CollectRep.MetricsData> responseMetrics) {
if (responseMetrics == null || responseMetrics.isEmpty()) {
sink.success();
} else {
List<String> jsons = new ArrayList<>(responseMetrics.size());
for (CollectRep.MetricsData metricsData : responseMetrics) {
String json = ProtoJsonUtil.toJsonStr(metricsData);
if (json != null) {
jsons.add(json);
}
}
sink.success(jsons);
}
}
};
timerDispatch.addJob(job, listener);
});
}
}

62
collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaDataExporter.java
View File

@@ -1,62 +0,0 @@
package com.usthe.collector.dispatch.export;
import com.usthe.collector.dispatch.DispatchProperties;
import com.usthe.common.entity.message.CollectRep;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.LongSerializer;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import java.util.Properties;
/**
* kafka采集数据消息发送
* @author tomsun28
* @date 2021/11/3 15:22
*/
@Configuration
@ConditionalOnProperty(prefix = "collector.dispatch.export.kafka",
name = "enabled", havingValue = "true", matchIfMissing = true)
@AutoConfigureAfter(value = {DispatchProperties.class})
@Slf4j
public class KafkaDataExporter implements DisposableBean {
KafkaProducer<Long, CollectRep.MetricsData> kafkaProducer;
DispatchProperties.ExportProperties.KafkaProperties kafkaProperties;
public KafkaDataExporter(DispatchProperties dispatchProperties) {
try {
kafkaProperties = dispatchProperties.getExport().getKafka();
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getServers());
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class);
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaMetricsDataSerializer.class);
kafkaProducer = new KafkaProducer<>(properties);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* 发送消息
* @param metricsData 指标组采集数据
*/
public void send(CollectRep.MetricsData metricsData) {
if (kafkaProducer != null) {
kafkaProducer.send(new ProducerRecord<>(kafkaProperties.getTopic(), metricsData.getId(), metricsData));
} else {
log.error("kafkaProducer is not enable");
}
}
@Override
public void destroy() throws Exception {
if (kafkaProducer != null) {
kafkaProducer.close();
}
}
}

18
collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaMetricsDataSerializer.java
View File

@@ -1,18 +0,0 @@
package com.usthe.collector.dispatch.export;
import com.usthe.common.entity.message.CollectRep;
import org.apache.kafka.common.serialization.Serializer;
/**
* MetricsData的序列化
* @author tomsun28
* @date 2021/11/3 16:14
*/
public class KafkaMetricsDataSerializer implements Serializer<CollectRep.MetricsData> {
@Override
public byte[] serialize(String topicName, CollectRep.MetricsData metricsData) {
return metricsData.toByteArray();
}
}

12
collector/server/src/main/java/com/usthe/collector/plugin/SameClass.java
View File

@@ -1,12 +0,0 @@
package com.usthe.collector.plugin;
/**
* @author tomsun28
* @date 2021/10/8 15:12
*/
public class SameClass {
public static String hello() {
return "hello collector";
}
}

20
collector/server/src/main/java/com/usthe/collector/plugin/TestPlugin.java
View File

@@ -1,20 +0,0 @@
package com.usthe.collector.plugin;
import com.usthe.plugin.sample.ExportDemo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/**
* @author tomsun28
* @date 2021/10/8 15:31
*/
@Component
@Slf4j
public class TestPlugin implements CommandLineRunner {
@Override
public void run(String... args) throws Exception {
log.info(SameClass.hello());
log.info(new ExportDemo().hello());
}
}

19
collector/server/src/main/resources/application.yml
View File

@@ -1,19 +0,0 @@
server:
port: 1157
spring:
application:
name: ${HOSTNAME:@collecor@}${PID}
profiles:
active: dev
jackson:
default-property-inclusion: NON_EMPTY
collector:
dispatch:
entrance:
etcd:
endpoints: http://139.198.109.64:2379
export:
kafka:
enabled: true
servers: 139.198.109.64:9092
topic: async-collect-data

6
collector/server/src/main/resources/banner.txt
View File

@@ -1,6 +0,0 @@
██████╗ ██████╗ ██╗ ██╗ ███████╗ ██████╗████████╗ ██████╗ ██████╗
██╔════╝██╔═══██╗██║ ██║ ██╔════╝██╔════╝╚══██╔══╝██╔═══██╗██╔══██╗
██║ ██║ ██║██║ ██║ █████╗ ██║ ██║ ██║ ██║██████╔╝
██║ ██║ ██║██║ ██║ ██╔══╝ ██║ ██║ ██║ ██║██╔══██╗ Profile: ${spring.profiles.active}
╚██████╗╚██████╔╝███████╗███████╗███████╗╚██████╗ ██║ ╚██████╔╝██║ ██║ Name: ${spring.application.name} Port: ${server.port} Pid: ${pid}
╚═════╝ ╚═════╝ ╚══════╝╚══════╝╚══════╝ ╚═════╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝

79
collector/server/src/main/resources/logback-spring.xml
View File

@@ -1,79 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true">
<springProperty scope="context" name="application_name" source="spring.application.name" defaultValue="collector"/>
<!-- 输出日志到控制台 ConsoleAppender -->
<appender name="ConsoleAppender" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<!--<pattern>%d %p (%file:%line\)- %m%n</pattern>-->
<!--格式化输出：%d:表示日期 %thread:表示线程名 %-5level:级别从左显示5个字符宽度 %msg:日志消息 %n:是换行符-->
<pattern>1-%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="SystemOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!-- 日志记录器的滚动策略，按日期，按大小记录 -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 归档的日志文件的路径。%d{yyyy-MM-dd}指定日期格式，%i指定索引 -->
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<!-- 除按日志记录之外，还配置了日志文件不能超过200M，若超过200M，日志文件会以索引0开始 -->
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>200MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- 追加方式记录日志 -->
<append>true</append>
<!-- 日志文件的格式 -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>===%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
</appender>
<appender name="ErrOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}-error.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>200MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- 追加方式记录日志 -->
<append>true</append>
<!-- 日志文件的格式 -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>===%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<!-- 此日志文件记录error及以上级别的 -->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
</appender>
<!--这个logger的设置是：举例在org.springframework包下面的所有输出日志必须级别level在info及以上级别才会被输出！-->
<!--这样可以避免输出一些spring框架的许多常见debug信息!-->
<logger name="org.springframework" level="info" />
<logger name="org.json" level="error"/>
<logger name="io.netty" level="info"/>
<logger name="org.slf4j" level="info"/>
<logger name="ch.qos.logback" level="info"/>
<!-- 生产环境配置 -->
<springProfile name="prod">
<root level="DEBUG">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
</springProfile>
<!-- 开发环境配置 -->
<springProfile name="dev">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
</springProfile>
</configuration>

0
collector/server/src/main/java/com/usthe/collector/collect/AbstractCollect.java → collector/src/main/java/com/usthe/collector/collect/AbstractCollect.java
View File

0
collector/server/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java → collector/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java
View File

0
collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java → collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java
View File

0
collector/server/src/main/java/com/usthe/collector/collect/icmp/IcmpCollectImpl.java → collector/src/main/java/com/usthe/collector/collect/icmp/IcmpCollectImpl.java
View File

0
collector/server/src/main/java/com/usthe/collector/collect/telnet/TelnetCollectImpl.java → collector/src/main/java/com/usthe/collector/collect/telnet/TelnetCollectImpl.java
View File

0
collector/server/src/main/java/com/usthe/collector/common/CollectorProperties.java → collector/src/main/java/com/usthe/collector/common/CollectorProperties.java
View File

0
collector/server/src/main/java/com/usthe/collector/common/cache/CacheCloseable.java → collector/src/main/java/com/usthe/collector/common/cache/CacheCloseable.java vendored
View File

0
collector/server/src/main/java/com/usthe/collector/common/cache/CacheDetectable.java → collector/src/main/java/com/usthe/collector/common/cache/CacheDetectable.java vendored
View File

0
collector/server/src/main/java/com/usthe/collector/common/cache/CacheIdentifier.java → collector/src/main/java/com/usthe/collector/common/cache/CacheIdentifier.java vendored
View File

0
collector/server/src/main/java/com/usthe/collector/common/cache/CommonCache.java → collector/src/main/java/com/usthe/collector/common/cache/CommonCache.java vendored
View File

0
collector/server/src/main/java/com/usthe/collector/common/cache/support/CommonJdbcConnect.java → collector/src/main/java/com/usthe/collector/common/cache/support/CommonJdbcConnect.java vendored
View File

0
collector/server/src/main/java/com/usthe/collector/common/http/CustomHttpRequestRetryHandler.java → collector/src/main/java/com/usthe/collector/common/http/CustomHttpRequestRetryHandler.java
View File

0
collector/server/src/main/java/com/usthe/collector/common/http/HttpClientPool.java → collector/src/main/java/com/usthe/collector/common/http/HttpClientPool.java
View File

0
collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpec.java → collector/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpec.java
View File

0
collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpecProvider.java → collector/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpecProvider.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/CollectDataDispatch.java → collector/src/main/java/com/usthe/collector/dispatch/CollectDataDispatch.java
View File

6
collector/server/src/main/java/com/usthe/collector/dispatch/CommonDispatcher.java → collector/src/main/java/com/usthe/collector/dispatch/CommonDispatcher.java
View File

@@ -1,6 +1,6 @@
package com.usthe.collector.dispatch;
import com.usthe.collector.dispatch.export.KafkaDataExporter;
import com.usthe.collector.dispatch.export.MetricsDataExporter;
import com.usthe.collector.dispatch.timer.Timeout;
import com.usthe.collector.dispatch.timer.TimerDispatch;
import com.usthe.collector.dispatch.timer.WheelTimerTask;
@@ -44,14 +44,14 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
/**
* kafka采集数据导出器
*/
private KafkaDataExporter kafkaDataExporter;
private MetricsDataExporter kafkaDataExporter;
/**
* 指标组任务与开始时间映射map
*/
private Map<String, MetricsTime> metricsTimeoutMonitorMap;
public CommonDispatcher(MetricsCollectorQueue jobRequestQueue, TimerDispatch timerDispatch,
KafkaDataExporter kafkaDataExporter, WorkerPool workerPool) {
MetricsDataExporter kafkaDataExporter, WorkerPool workerPool) {
this.kafkaDataExporter = kafkaDataExporter;
this.jobRequestQueue = jobRequestQueue;
this.timerDispatch = timerDispatch;

0
collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java → collector/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/DispatchProperties.java → collector/src/main/java/com/usthe/collector/dispatch/DispatchProperties.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java → collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollectorQueue.java → collector/src/main/java/com/usthe/collector/dispatch/MetricsCollectorQueue.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/MetricsTaskDispatch.java → collector/src/main/java/com/usthe/collector/dispatch/MetricsTaskDispatch.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/WorkerPool.java → collector/src/main/java/com/usthe/collector/dispatch/WorkerPool.java
View File

83
collector/src/main/java/com/usthe/collector/dispatch/entrance/internal/CollectJobService.java Normal file
View File

@@ -0,0 +1,83 @@
package com.usthe.collector.dispatch.entrance.internal;
import com.usthe.collector.dispatch.timer.TimerDispatch;
import com.usthe.common.entity.job.Job;
import com.usthe.common.entity.message.CollectRep;
import com.usthe.common.util.SnowFlakeIdGenerator;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* 采集job管理提供api接口
* @author tomsun28
* @date 2021/11/6 13:58
*/
@Service
@Slf4j
public class CollectJobService {
@Autowired
private TimerDispatch timerDispatch;
/**
* 执行一次性采集任务,获取采集数据响应
* @param job 采集任务详情
* @return 采集结果
*/
public List<CollectRep.MetricsData> collectSyncJobData(Job job) {
final List<CollectRep.MetricsData> metricsData = new LinkedList<>();
final CountDownLatch countDownLatch = new CountDownLatch(1);
CollectResponseEventListener listener = new CollectResponseEventListener() {
@Override
public void response(List<CollectRep.MetricsData> responseMetrics) {
if (responseMetrics != null) {
metricsData.addAll(responseMetrics);
}
countDownLatch.countDown();
}
};
timerDispatch.addJob(job, listener);
try {
countDownLatch.await(100, TimeUnit.SECONDS);
} catch (Exception e) {
log.info("同步任务运行100秒无响应，返回");
}
return metricsData;
}
/**
* 下发周期性异步采集任务
* @param job 采集任务详情
* @return long 任务ID
*/
public long addAsyncCollectJob(Job job) {
long jobId = SnowFlakeIdGenerator.generateId();
job.setId(jobId);
timerDispatch.addJob(job, null);
return jobId;
}
/**
* 更新已经下发的周期性异步采集任务
* @param modifyJob 采集任务详情
*/
public void updateAsyncCollectJob(Job modifyJob) {
timerDispatch.deleteJob(modifyJob.getId(), true);
timerDispatch.addJob(modifyJob, null);
}
/**
* 取消周期性异步采集任务
* @param jobId 任务ID
*/
public void cancelAsyncCollectJob(Long jobId) {
timerDispatch.deleteJob(jobId, true);
}
}

4
collector/server/src/main/java/com/usthe/collector/dispatch/entrance/http/CollectResponseEventListener.java → collector/src/main/java/com/usthe/collector/dispatch/entrance/internal/CollectResponseEventListener.java
View File

@@ -1,4 +1,4 @@
package com.usthe.collector.dispatch.entrance.http;
package com.usthe.collector.dispatch.entrance.internal;
import com.usthe.common.entity.message.CollectRep;
@@ -16,5 +16,5 @@ public interface CollectResponseEventListener extends EventListener {
* 采集任务完成结果通知
* @param responseMetrics 响应数据
*/
public default void response(List<CollectRep.MetricsData> responseMetrics) {}
default void response(List<CollectRep.MetricsData> responseMetrics) {}
}

56
collector/src/main/java/com/usthe/collector/dispatch/export/MetricsDataExporter.java Normal file
View File

@@ -0,0 +1,56 @@
package com.usthe.collector.dispatch.export;
import com.usthe.common.entity.message.CollectRep;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* 采集数据消息发送
* @author tomsun28
* @date 2021/11/3 15:22
*/
@Component
@Slf4j
public class MetricsDataExporter implements DisposableBean {
private final LinkedBlockingQueue<CollectRep.MetricsData> metricsDataToAlertQueue;
private final LinkedBlockingQueue<CollectRep.MetricsData> metricsDataToWarehouseInfluxQueue;
private final LinkedBlockingQueue<CollectRep.MetricsData> metricsDataToWarehouseRedisQueue;
public MetricsDataExporter() {
metricsDataToAlertQueue = new LinkedBlockingQueue<>();
metricsDataToWarehouseInfluxQueue = new LinkedBlockingQueue<>();
metricsDataToWarehouseRedisQueue = new LinkedBlockingQueue<>();
}
public CollectRep.MetricsData pollAlertMetricsData() throws InterruptedException {
return metricsDataToAlertQueue.poll(2, TimeUnit.SECONDS);
}
public CollectRep.MetricsData pollWarehouseInfluxMetricsData() throws InterruptedException {
return metricsDataToAlertQueue.poll(2, TimeUnit.SECONDS);
}
public CollectRep.MetricsData pollWarehouseRedisMetricsData() throws InterruptedException {
return metricsDataToWarehouseRedisQueue.poll(2, TimeUnit.SECONDS);
}
/**
* 发送消息
* @param metricsData 指标组采集数据
*/
public void send(CollectRep.MetricsData metricsData) {
metricsDataToAlertQueue.offer(metricsData);
metricsDataToWarehouseInfluxQueue.offer(metricsData);
metricsDataToWarehouseRedisQueue.offer(metricsData);
}
@Override
public void destroy() throws Exception {
metricsDataToAlertQueue.clear();
}
}

0
collector/server/src/main/java/com/usthe/collector/dispatch/timer/HashedWheelTimer.java → collector/src/main/java/com/usthe/collector/dispatch/timer/HashedWheelTimer.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timeout.java → collector/src/main/java/com/usthe/collector/dispatch/timer/Timeout.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timer.java → collector/src/main/java/com/usthe/collector/dispatch/timer/Timer.java
View File

2
collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatch.java → collector/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatch.java
View File

@@ -1,7 +1,7 @@
package com.usthe.collector.dispatch.timer;
import com.usthe.collector.dispatch.entrance.http.CollectResponseEventListener;
import com.usthe.collector.dispatch.entrance.internal.CollectResponseEventListener;
import com.usthe.common.entity.job.Job;
import com.usthe.common.entity.message.CollectRep;

2
collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatcher.java → collector/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatcher.java
View File

@@ -1,6 +1,6 @@
package com.usthe.collector.dispatch.timer;
import com.usthe.collector.dispatch.entrance.http.CollectResponseEventListener;
import com.usthe.collector.dispatch.entrance.internal.CollectResponseEventListener;
import com.usthe.common.entity.job.Job;
import com.usthe.common.entity.message.CollectRep;
import org.springframework.stereotype.Component;

0
collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerTask.java → collector/src/main/java/com/usthe/collector/dispatch/timer/TimerTask.java
View File

0
collector/server/src/main/java/com/usthe/collector/dispatch/timer/WheelTimerTask.java → collector/src/main/java/com/usthe/collector/dispatch/timer/WheelTimerTask.java
View File

0
collector/server/src/main/java/com/usthe/collector/util/CollectorConstants.java → collector/src/main/java/com/usthe/collector/util/CollectorConstants.java
View File

0
collector/server/src/main/java/com/usthe/collector/util/JsonPathParser.java → collector/src/main/java/com/usthe/collector/util/JsonPathParser.java
View File

0
collector/server/src/main/java/com/usthe/collector/util/SpringContextHolder.java → collector/src/main/java/com/usthe/collector/util/SpringContextHolder.java
View File

9
collector/src/main/resources/META-INF/spring.factories Normal file
View File

@@ -0,0 +1,9 @@
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.usthe.collector.dispatch.timer.TimerDispatcher,\
com.usthe.collector.dispatch.CommonDispatcher,\
com.usthe.collector.dispatch.DispatchProperties,\
com.usthe.collector.dispatch.MetricsCollectorQueue,\
com.usthe.collector.dispatch.WorkerPool,\
com.usthe.collector.dispatch.entrance.internal.CollectJobService,\
com.usthe.collector.dispatch.export.MetricsDataExporter,\
com.usthe.collector.util.SpringContextHolder

0
collector/server/src/test/java/com/usthe/collector/collect/telnet/TelnetCollectImplTest.java → collector/src/test/java/com/usthe/collector/collect/telnet/TelnetCollectImplTest.java
View File