From 7286263836211bf461b932433d627f82a60cd2dd Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Wed, 10 Nov 2021 20:07:27 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BF=81=E7=A7=BBcollector=EF=BC=8C=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E5=88=B0=E6=95=B4=E4=B8=AA=E5=B7=A5=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 37 + LICENSE | 207 ++ README.md | 6 + alerter/pom.xml | 18 + collector/README.md | 42 + collector/plugins/pom.xml | 19 + collector/plugins/sample-plugin/pom.xml | 48 + .../com/usthe/collector/plugin/SameClass.java | 12 + .../com/usthe/plugin/sample/ExportDemo.java | 14 + collector/pom.xml | 18 + collector/server/pom.xml | 118 + .../java/com/usthe/collector/Collector.java | 38 + .../collector/collect/AbstractCollect.java | 23 + .../collect/database/JdbcCommonCollect.java | 188 + .../collect/http/HttpCollectImpl.java | 247 ++ .../collector/common/CollectorProperties.java | 15 + .../common/cache/CacheCloseable.java | 14 + .../common/cache/CacheDetectable.java | 15 + .../common/cache/CacheIdentifier.java | 23 + .../collector/common/cache/CommonCache.java | 218 ++ .../cache/support/CommonJdbcConnect.java | 55 + .../http/CustomHttpRequestRetryHandler.java | 116 + .../collector/common/http/HttpClientPool.java | 170 + .../common/http/IgnoreReqCookieSpec.java | 29 + .../http/IgnoreReqCookieSpecProvider.java | 32 + .../dispatch/CollectDataDispatch.java | 23 + .../collector/dispatch/CommonDispatcher.java | 187 + .../dispatch/DispatchConfiguration.java | 21 + .../collector/dispatch/DispatchConstants.java | 65 + .../dispatch/DispatchProperties.java | 225 ++ .../collector/dispatch/MetricsCollect.java | 166 + .../dispatch/MetricsCollectorQueue.java | 32 + .../dispatch/MetricsTaskDispatch.java | 17 + .../usthe/collector/dispatch/WorkerPool.java | 54 + .../entrance/http/CollectJobController.java | 25 + .../dispatch/export/KafkaDataExporter.java | 54 + .../export/KafkaMetricsDataSerializer.java | 18 + .../dispatch/timer/HashedWheelTimer.java | 809 +++++ .../collector/dispatch/timer/Timeout.java | 57 + .../usthe/collector/dispatch/timer/Timer.java | 58 + .../dispatch/timer/TimerDispatch.java | 35 + .../dispatch/timer/TimerDispatcher.java | 75 + .../collector/dispatch/timer/TimerTask.java | 36 + .../dispatch/timer/WheelTimerJob.java | 30 + .../com/usthe/collector/plugin/SameClass.java | 12 + .../usthe/collector/plugin/TestPlugin.java | 20 + .../collector/util/SpringContextHolder.java | 48 + .../server/src/main/resources/application.yml | 16 + .../server/src/main/resources/banner.txt | 6 + .../src/main/resources/logback-spring.xml | 79 + common/pom.xml | 35 + .../usthe/common/entity/job/Configmap.java | 35 + .../java/com/usthe/common/entity/job/Job.java | 162 + .../com/usthe/common/entity/job/Metrics.java | 70 + .../entity/job/protocol/HttpProtocol.java | 93 + .../entity/job/protocol/IcmpProtocol.java | 23 + .../entity/job/protocol/JdbcProtocol.java | 38 + .../entity/job/protocol/TcpUdpProtocol.java | 30 + .../common/entity/message/CollectRep.java | 3225 +++++++++++++++++ .../java/com/usthe/common/util/GsonUtil.java | 41 + common/src/main/message/collect_rep.proto | 46 + manager/pom.xml | 18 + pom.xml | 129 + 63 files changed, 7835 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 alerter/pom.xml create mode 100644 collector/README.md create mode 100644 collector/plugins/pom.xml create mode 100644 collector/plugins/sample-plugin/pom.xml create mode 100644 collector/plugins/sample-plugin/src/main/java/com/usthe/collector/plugin/SameClass.java create mode 100644 collector/plugins/sample-plugin/src/main/java/com/usthe/plugin/sample/ExportDemo.java create mode 100644 collector/pom.xml create mode 100644 collector/server/pom.xml create mode 100644 collector/server/src/main/java/com/usthe/collector/Collector.java create mode 100644 collector/server/src/main/java/com/usthe/collector/collect/AbstractCollect.java create mode 100644 collector/server/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java create mode 100644 collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/CollectorProperties.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/cache/CacheCloseable.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/cache/CacheDetectable.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/cache/CacheIdentifier.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/cache/CommonCache.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/cache/support/CommonJdbcConnect.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/http/CustomHttpRequestRetryHandler.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/http/HttpClientPool.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpec.java create mode 100644 collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpecProvider.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/CollectDataDispatch.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/CommonDispatcher.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConfiguration.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/DispatchProperties.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollectorQueue.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/MetricsTaskDispatch.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/WorkerPool.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/entrance/http/CollectJobController.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaDataExporter.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaMetricsDataSerializer.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/HashedWheelTimer.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timeout.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timer.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatch.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatcher.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerTask.java create mode 100644 collector/server/src/main/java/com/usthe/collector/dispatch/timer/WheelTimerJob.java create mode 100644 collector/server/src/main/java/com/usthe/collector/plugin/SameClass.java create mode 100644 collector/server/src/main/java/com/usthe/collector/plugin/TestPlugin.java create mode 100644 collector/server/src/main/java/com/usthe/collector/util/SpringContextHolder.java create mode 100644 collector/server/src/main/resources/application.yml create mode 100644 collector/server/src/main/resources/banner.txt create mode 100644 collector/server/src/main/resources/logback-spring.xml create mode 100644 common/pom.xml create mode 100644 common/src/main/java/com/usthe/common/entity/job/Configmap.java create mode 100644 common/src/main/java/com/usthe/common/entity/job/Job.java create mode 100644 common/src/main/java/com/usthe/common/entity/job/Metrics.java create mode 100644 common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java create mode 100644 common/src/main/java/com/usthe/common/entity/job/protocol/IcmpProtocol.java create mode 100644 common/src/main/java/com/usthe/common/entity/job/protocol/JdbcProtocol.java create mode 100644 common/src/main/java/com/usthe/common/entity/job/protocol/TcpUdpProtocol.java create mode 100644 common/src/main/java/com/usthe/common/entity/message/CollectRep.java create mode 100644 common/src/main/java/com/usthe/common/util/GsonUtil.java create mode 100644 common/src/main/message/collect_rep.proto create mode 100644 manager/pom.xml create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84ee25c --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +Thumbs.db +.DS_Store +.gradle +build/ +out/ +micronaut-cli.yml +.mvn/ +mvnw +mvnw.bat +.log + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +### VS Code ### +.vscode/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b018625 --- /dev/null +++ b/LICENSE @@ -0,0 +1,207 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, and + distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by the + copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all other + entities that control, are controlled by, or are under common control with + that entity. For the purposes of this definition, "control" means (i) the + power, direct or indirect, to cause the direction or management of such + entity, whether by contract or otherwise, or (ii) ownership of + fifty percent (50%) or more of the outstanding shares, or (iii) beneficial + ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity exercising + permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation source, + and configuration files. + + "Object" form shall mean any form resulting from mechanical transformation + or translation of a Source form, including but not limited to compiled + object code, generated documentation, and conversions to + other media types. + + "Work" shall mean the work of authorship, whether in Source or Object + form, made available under the License, as indicated by a copyright notice + that is included in or attached to the work (an example is provided in the + Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object form, + that is based on (or derived from) the Work and for which the editorial + revisions, annotations, elaborations, or other modifications represent, + as a whole, an original work of authorship. For the purposes of this + License, Derivative Works shall not include works that remain separable + from, or merely link (or bind by name) to the interfaces of, the Work and + Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including the original + version of the Work and any modifications or additions to that Work or + Derivative Works thereof, that is intentionally submitted to Licensor for + inclusion in the Work by the copyright owner or by an individual or + Legal Entity authorized to submit on behalf of the copyright owner. + For the purposes of this definition, "submitted" means any form of + electronic, verbal, or written communication sent to the Licensor or its + representatives, including but not limited to communication on electronic + mailing lists, source code control systems, and issue tracking systems + that are managed by, or on behalf of, the Licensor for the purpose of + discussing and improving the Work, but excluding communication that is + conspicuously marked or otherwise designated in writing by the copyright + owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity on + behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. + + Subject to the terms and conditions of this License, each Contributor + hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable copyright license to reproduce, prepare + Derivative Works of, publicly display, publicly perform, sublicense, + and distribute the Work and such Derivative Works in + Source or Object form. + +3. Grant of Patent License. + + Subject to the terms and conditions of this License, each Contributor + hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable (except as stated in this section) patent + license to make, have made, use, offer to sell, sell, import, and + otherwise transfer the Work, where such license applies only to those + patent claims licensable by such Contributor that are necessarily + infringed by their Contribution(s) alone or by combination of their + Contribution(s) with the Work to which such Contribution(s) was submitted. + If You institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work or a + Contribution incorporated within the Work constitutes direct or + contributory patent infringement, then any patent licenses granted to + You under this License for that Work shall terminate as of the date such + litigation is filed. + +4. Redistribution. + + You may reproduce and distribute copies of the Work or Derivative Works + thereof in any medium, with or without modifications, and in Source or + Object form, provided that You meet the following conditions: + + 1. You must give any other recipients of the Work or Derivative Works a + copy of this License; and + + 2. You must cause any modified files to carry prominent notices stating + that You changed the files; and + + 3. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices from + the Source form of the Work, excluding those notices that do not pertain + to any part of the Derivative Works; and + + 4. If the Work includes a "NOTICE" text file as part of its distribution, + then any Derivative Works that You distribute must include a readable copy + of the attribution notices contained within such NOTICE file, excluding + those notices that do not pertain to any part of the Derivative Works, + in at least one of the following places: within a NOTICE text file + distributed as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, within a + display generated by the Derivative Works, if and wherever such + third-party notices normally appear. The contents of the NOTICE file are + for informational purposes only and do not modify the License. + You may add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the Work, + provided that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and may + provide additional or different license terms and conditions for use, + reproduction, or distribution of Your modifications, or for any such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions + stated in this License. + +5. Submission of Contributions. + + Unless You explicitly state otherwise, any Contribution intentionally + submitted for inclusion in the Work by You to the Licensor shall be under + the terms and conditions of this License, without any additional + terms or conditions. Notwithstanding the above, nothing herein shall + supersede or modify the terms of any separate license agreement you may + have executed with Licensor regarding such Contributions. + +6. Trademarks. + + This License does not grant permission to use the trade names, trademarks, + service marks, or product names of the Licensor, except as required for + reasonable and customary use in describing the origin of the Work and + reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + + Unless required by applicable law or agreed to in writing, Licensor + provides the Work (and each Contributor provides its Contributions) + on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + either express or implied, including, without limitation, any warranties + or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS + FOR A PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any risks + associated with Your exercise of permissions under this License. + +8. Limitation of Liability. + + In no event and under no legal theory, whether in tort + (including negligence), contract, or otherwise, unless required by + applicable law (such as deliberate and grossly negligent acts) or agreed + to in writing, shall any Contributor be liable to You for damages, + including any direct, indirect, special, incidental, or consequential + damages of any character arising as a result of this License or out of + the use or inability to use the Work (including but not limited to damages + for loss of goodwill, work stoppage, computer failure or malfunction, + or any and all other commercial damages or losses), even if such + Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + + While redistributing the Work or Derivative Works thereof, You may choose + to offer, and charge a fee for, acceptance of support, warranty, + indemnity, or other liability obligations and/or rights consistent with + this License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of any + other Contributor, and only if You agree to indemnify, defend, and hold + each Contributor harmless for any liability incurred by, or claims + asserted against, such Contributor by reason of your accepting any such + warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + + To apply the Apache License to your work, attach the following boilerplate + notice, with the fields enclosed by brackets "[]" replaced with your own + identifying information. (Don't include the brackets!) The text should be + enclosed in the appropriate comment syntax for the file format. We also + recommend that a file or class name and description of purpose be included + on the same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright tomsun28 usthe.com. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing + permissions and limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..aa42ce7 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +## monitor + +- 提供监控管理服务 +- 提供监控任务下发服务 +- 提供监控数据查询服务 +- 告警服务 \ No newline at end of file diff --git a/alerter/pom.xml b/alerter/pom.xml new file mode 100644 index 0000000..1c6b56b --- /dev/null +++ b/alerter/pom.xml @@ -0,0 +1,18 @@ + + + + monitor + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + alerter + + + + + + \ No newline at end of file diff --git a/collector/README.md b/collector/README.md new file mode 100644 index 0000000..f9ecdd2 --- /dev/null +++ b/collector/README.md @@ -0,0 +1,42 @@ +### TanCloud Collector + +* 操作系统 + * Linux + * Windows + * Ubuntu + * CentOs +* 数据库 + * Mysql + * Oracle + * PostgreSQL +* 中间件 + * Kafka + * Zookeeper + * RocketMq + * Etcd +* 云原生 + * Docker + * Kubernetes + * Istio +* 应用服务 + * Tomcat + * Jetty + * Http + * Ping + * 服务端口 + +#### HELP + +1. ARK插件类隔离未生效 +> 注意需构建在jdk1.8环境中运行 +> 插件是否配置导入并配置正确 +> 本地DEBUG时需单独IDEA打开运行collector工程，不能将plugin和collector在同一工程打开运行 + +2. metaspace元空间内存占用多或溢出 +> 建议调整JVM参数 ```-Dsun.reflect.inflationThreshold=100000``` +> 由于使用太多反射，超过参数`inflationThreshold`默认值15阈值导致触发JVM反射优化(加快反射速度), +> 反射获取类信息由使用*JNI存取器**膨胀(Inflation)* +> 为*反射每个方法生成一个类加载器DelegatingClassLoader和Java类MethodAccessor*. +> 动态加载的字节码导致PermGen持续增长. + + diff --git a/collector/plugins/pom.xml b/collector/plugins/pom.xml new file mode 100644 index 0000000..14c317b --- /dev/null +++ b/collector/plugins/pom.xml @@ -0,0 +1,19 @@ + + + + collector + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + plugins + pom + + sample-plugin + + + + \ No newline at end of file diff --git a/collector/plugins/sample-plugin/pom.xml b/collector/plugins/sample-plugin/pom.xml new file mode 100644 index 0000000..6ce90c0 --- /dev/null +++ b/collector/plugins/sample-plugin/pom.xml @@ -0,0 +1,48 @@ + + + + plugins + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + sample-plugin + + + + + + com.alipay.sofa + sofa-ark-plugin-maven-plugin + 1.1.6 + + + default-cli + + ark-plugin + + + + + 2000 + + + + + + com.com.usthe.plugin.sample.ExportDemo + + + + + + + + + + + + \ No newline at end of file diff --git a/collector/plugins/sample-plugin/src/main/java/com/usthe/collector/plugin/SameClass.java b/collector/plugins/sample-plugin/src/main/java/com/usthe/collector/plugin/SameClass.java new file mode 100644 index 0000000..b6bbe94 --- /dev/null +++ b/collector/plugins/sample-plugin/src/main/java/com/usthe/collector/plugin/SameClass.java @@ -0,0 +1,12 @@ +package com.usthe.collector.plugin; + +/** + * @author tomsun28 + * @date 2021/10/8 15:12 + */ +public class SameClass { + + public static String hello() { + return "hello plugin"; + } +} diff --git a/collector/plugins/sample-plugin/src/main/java/com/usthe/plugin/sample/ExportDemo.java b/collector/plugins/sample-plugin/src/main/java/com/usthe/plugin/sample/ExportDemo.java new file mode 100644 index 0000000..c3e163d --- /dev/null +++ b/collector/plugins/sample-plugin/src/main/java/com/usthe/plugin/sample/ExportDemo.java @@ -0,0 +1,14 @@ +package com.usthe.plugin.sample; + +import com.usthe.collector.plugin.SameClass; + +/** + * @author tomsun28 + * @date 2021/10/8 15:11 + */ +public class ExportDemo { + + public String hello() { + return SameClass.hello(); + } +} diff --git a/collector/pom.xml b/collector/pom.xml new file mode 100644 index 0000000..e30de1e --- /dev/null +++ b/collector/pom.xml @@ -0,0 +1,18 @@ + + + + monitor + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + collector + pom + + server + plugins + + \ No newline at end of file diff --git a/collector/server/pom.xml b/collector/server/pom.xml new file mode 100644 index 0000000..c21a8c3 --- /dev/null +++ b/collector/server/pom.xml @@ -0,0 +1,118 @@ + + + + collector + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + server + + + + + org.springframework.boot + spring-boot-starter-webflux + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + com.alipay.sofa + sofa-ark-springboot-starter + 1.1.6 + + + + com.usthe.tancloud + common + 1.0-SNAPSHOT + + + + io.etcd + jetcd-core + 0.5.10 + + + + org.apache.kafka + kafka-clients + 3.0.0 + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + + com.googlecode.concurrentlinkedhashmap + concurrentlinkedhashmap-lru + 1.4.2 + + + + com.usthe.tancloud + common + 1.0-SNAPSHOT + + + com.google.guava + guava + 31.0.1-jre + + + com.google.code.gson + gson + 2.8.8 + + + com.googlecode.aviator + aviator + 5.2.7 + + + + com.usthe.tancloud + sample-plugin + 1.0-SNAPSHOT + + + + + + + com.alipay.sofa + sofa-ark-maven-plugin + 1.1.6 + + + default-cli + + + + repackage + + + + + ./target + + + executable + + + + + + + + \ No newline at end of file diff --git a/collector/server/src/main/java/com/usthe/collector/Collector.java b/collector/server/src/main/java/com/usthe/collector/Collector.java new file mode 100644 index 0000000..ac67dc5 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/Collector.java @@ -0,0 +1,38 @@ +package com.usthe.collector; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; + +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; + +/** + * collector start + * @author tomsun28 + * @date 2021/10/7 18:02 + */ +@SpringBootApplication(exclude = {KafkaAutoConfiguration.class}) +@Slf4j +public class Collector { + public static void main(String[] args) { + SpringApplication.run(Collector.class, args); +// DatagramSocket s = new DatagramSocket(); +// +// /** 2、提供数据，封装打包 ---DatagramPacket(byte[] buf, int length, InetAddress address, int port) */ +// +// byte[] bs = "正在使用UDP发送--我是数据! ".getBytes(); +// DatagramPacket dp = new DatagramPacket(bs, bs.length, InetAddress.getByName("192.168.1.189"), 8070); +// +// /** 3、使用send发送 */ +// try { +// s.send(dp); +// s.receive(dp) +// } catch (IOException e) { +// System.out.println("发送失败： "); +// e.printStackTrace(); +// } + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/collect/AbstractCollect.java b/collector/server/src/main/java/com/usthe/collector/collect/AbstractCollect.java new file mode 100644 index 0000000..58eba4a --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/collect/AbstractCollect.java @@ -0,0 +1,23 @@ +package com.usthe.collector.collect; + + +import com.usthe.common.entity.job.Metrics; +import com.usthe.common.entity.message.CollectRep; + +/** + * 具体的指标组采集实现抽象类 + * @author tomsun28 + * @date 2021/11/4 9:35 + */ +public abstract class AbstractCollect { + + /** + * 真正的采集实现接口 + * @param builder response builder + * @param appId 应用监控ID + * @param app 应用类型 + * @param metrics 指标组配置 + * return response builder + */ + public abstract void collect(CollectRep.MetricsData.Builder builder, long appId, String app, Metrics metrics); +} diff --git a/collector/server/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java b/collector/server/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java new file mode 100644 index 0000000..d9883ad --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/collect/database/JdbcCommonCollect.java @@ -0,0 +1,188 @@ +package com.usthe.collector.collect.database; + +import com.usthe.collector.common.cache.CacheIdentifier; +import com.usthe.collector.common.cache.CommonCache; +import com.usthe.collector.common.cache.support.CommonJdbcConnect; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +/** + * 数据库JDBC通用查询 + * @author tomsun28 + * @date 2021/9/1 21:37 + */ +@Slf4j +public class JdbcCommonCollect { + + + private Statement getConnection(String username, String password, String url) { + CacheIdentifier identifier = CacheIdentifier.builder() + .ip(url) + .username(username).password(password).build(); + Optional cacheOption = CommonCache.getInstance().getCache(identifier, true); + Statement statement = null; + if (cacheOption.isPresent()) { + CommonJdbcConnect jdbcConnect = (CommonJdbcConnect) cacheOption.get(); + try { + statement = jdbcConnect.getConnection().createStatement(); + // 设置查询超时时间10秒 + statement.setQueryTimeout(10); + // 设置查询最大行数1000行 + statement.setMaxRows(1000); + } catch (Exception e) { + log.info("The jdbc connect form cache, create statement error: {}", e.getMessage()); + try { + if (statement != null) { + statement.close(); + } + jdbcConnect.close(); + statement = null; + } catch (Exception e2) { + log.error(e2.getMessage()); + } + CommonCache.getInstance().removeCache(identifier); + } + } + if (statement != null) { + return statement; + } + // 复用失败则新建连接 + try { + Connection connection = DriverManager.getConnection(url, username, password); + statement = connection.createStatement(); + // 设置查询超时时间10秒 + statement.setQueryTimeout(10); + // 设置查询最大行数1000行 + statement.setMaxRows(1000); + CommonJdbcConnect jdbcConnect = new CommonJdbcConnect(connection); + CommonCache.getInstance().addCache(identifier, jdbcConnect, 10000L); + } catch (SQLException sqlException) { + log.error("Jdbc sql error: {}, code: {}.", sqlException.getMessage(), sqlException.getErrorCode(), sqlException); + } catch (Exception e) { + log.error("Jdbc error: {}.", e.getMessage(), e); + } + return statement; + } + + /** + * 查询一行数据, 通过查询返回结果集的列名称，和查询的字段映射 + * eg: + * 查询字段：one tow three four + * 查询SQL：select one, tow, three, four from book limit 1; + * @param statement 执行器 + * @param sql sql + * @param columns 查询的列头(一般是数据库表字段，也可能包含特殊字段,eg: responseTime) + * @throws Exception when error happen + */ + private List> queryOneRow(Statement statement, String sql, List columns) throws Exception { + long startTime = System.currentTimeMillis(); + statement.setMaxRows(1); + ResultSet resultSet = statement.executeQuery(sql); + List> rowList = new LinkedList<>(); + try { + if (resultSet.next()) { + LinkedList rows = new LinkedList<>(); + for (String column : columns) { + if ("responseTime".equals(column)) { + long time = System.currentTimeMillis() - startTime; + rows.add(String.valueOf(time)); + } else { + String value = resultSet.getString(column); + value = value == null ? "" : value; + rows.add(value); + } + } + rowList.add(rows); + } + } finally { + resultSet.close(); + } + return rowList; + } + + /** + * 查询一行数据, 通过查询的两列数据(key-value)，key和查询的字段匹配，value为查询字段的值 + * eg: + * 查询字段：one tow three four + * 查询SQL：select key, value from book; + * 返回的key映射查询字段 + * @param statement 执行器 + * @param sql sql + * @param columns 查询的列头(一般是数据库表字段，也可能包含特殊字段,eg: responseTime) + * @throws Exception when error happen + */ + private List> queryOneRowByMatchTwoColumns(Statement statement, String sql, List columns) throws Exception { + long startTime = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + List> rowList = new LinkedList<>(); + try { + HashMap values = new HashMap<>(columns.size()); + while (resultSet.next()) { + if (resultSet.getString(1) != null) { + values.put(resultSet.getString(1).toLowerCase(), resultSet.getString(2)); + } + } + LinkedList rows = new LinkedList<>(); + for (String column : columns) { + if ("responseTime".equals(column)) { + long time = System.currentTimeMillis() - startTime; + rows.add(String.valueOf(time)); + } else { + String value = values.get(column.toLowerCase()); + value = value == null ? "" : value; + rows.add(value); + } + } + rowList.add(rows); + } finally { + resultSet.close(); + } + return rowList; + } + + /** + * 查询多行数据, 通过查询返回结果集的列名称，和查询的字段映射 + * eg: + * 查询字段：one tow three four + * 查询SQL：select one, tow, three, four from book limit 1; + * @param statement 执行器 + * @param sql sql + * @param columns 查询的列头(一般是数据库表字段，也可能包含特殊字段,eg: responseTime) + * @throws Exception when error happen + */ + private List> queryMultiRow(Statement statement, String sql, List columns) throws Exception { + long startTime = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + List> rowList = new LinkedList<>(); + try { + while (resultSet.next()) { + LinkedList rows = new LinkedList<>(); + for (String column : columns) { + if ("responseTime".equals(column)) { + long time = System.currentTimeMillis() - startTime; + rows.add(String.valueOf(time)); + } else { + String value = resultSet.getString(column); + value = value == null ? "" : value; + rows.add(value); + } + } + rowList.add(rows); + } + } finally { + resultSet.close(); + } + return rowList; + } + + +} diff --git a/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java b/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java new file mode 100644 index 0000000..df040e7 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java @@ -0,0 +1,247 @@ +package com.usthe.collector.collect.http; + +import com.usthe.collector.collect.AbstractCollect; +import com.usthe.collector.common.http.HttpClientPool; +import com.usthe.collector.dispatch.DispatchConstants; +import com.usthe.common.entity.job.Metrics; +import com.usthe.common.entity.job.protocol.HttpProtocol; +import com.usthe.common.entity.message.CollectRep; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; +import org.springframework.http.HttpMethod; + +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +/** + * http https 采集实现类 + * @author tomsun28 + * @date 2021/11/4 15:37 + */ +@Slf4j +public class HttpCollectImpl extends AbstractCollect { + + private HttpCollectImpl() {} + + public static HttpCollectImpl getInstance() { + return Singleton.INSTANCE; + } + + @Override + public void collect(CollectRep.MetricsData.Builder builder, + long appId, String app, Metrics metrics) { + // 简单校验必有参数 + if (metrics == null || metrics.getHttp() == null) { + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg("Http/Https collect must has http params"); + return; + } + HttpContext httpContext = createHttpContext(metrics.getHttp()); + HttpUriRequest request = createHttpRequest(metrics.getHttp()); + try { + CloseableHttpResponse response = HttpClientPool.getHttpClient() + .execute(request, httpContext); + int statusCode = response.getStatusLine().getStatusCode(); + log.debug("http response status: {}", statusCode); + if (statusCode < HttpStatus.SC_OK || statusCode >= HttpStatus.SC_MULTIPLE_CHOICES) { + // 1XX 3XX 4XX 5XX 状态码 失败 + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg("statusCode: " + statusCode); + return; + } else { + // 2xx 状态码 成功 + String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + // 根据不同的解析方式解析 + if (resp == null || "".equals(resp)) { + log.info("http response entity is empty, status: {}.", statusCode); + builder.setCode(CollectRep.Code.SUCCESS); + builder.setMsg("statusCode: " + statusCode + ",entity empty."); + return; + } + String parseType = metrics.getHttp().getParseType(); + try { + if (DispatchConstants.PARSE_DEFAULT.equals(parseType)) { + parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + } else if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) { + parseResponseByPrometheus(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + } else if (DispatchConstants.PARSE_JSON_PATH.equals(parseType)) { + parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + } else if (DispatchConstants.PARSE_XML_PATH.equals(parseType)) { + parseResponseByXmlPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + } else { + parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder); + } + } catch (Exception e) { + log.info("parse error: {}.", e.getMessage(), e); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg("parse response data error."); + return; + } + } + } catch (ClientProtocolException e1) { + log.error(e1.getMessage(), e1); + } catch (UnknownHostException e2) { + // 对端不可达 + log.info(e2.getMessage()); + builder.setCode(CollectRep.Code.UN_REACHABLE); + builder.setMsg("unknown host"); + return; + } catch (InterruptedIOException | ConnectException | SSLException e3) { + // 对端连接失败 + log.info(e3.getMessage()); + builder.setCode(CollectRep.Code.UN_CONNECTABLE); + builder.setMsg(e3.getMessage()); + return; + } catch (IOException e4) { + // 其它IO异常 + log.info(e4.getMessage()); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(e4.getMessage()); + return; + } catch (Exception e) { + // 其它异常 + log.error(e.getMessage(), e); + builder.setCode(CollectRep.Code.FAIL); + builder.setMsg(e.getMessage()); + return; + } finally { + if (request != null) { + request.abort(); + } + } + } + + private void parseResponseByXmlPath(String resp, List preFields, HttpProtocol http, CollectRep.MetricsData.Builder builder) { + + } + + private void parseResponseByJsonPath(String resp, List preFields, HttpProtocol http, CollectRep.MetricsData.Builder builder) { + + } + + private void parseResponseByPrometheus(String resp, List preFields, HttpProtocol http, CollectRep.MetricsData.Builder builder) { + + } + + private void parseResponseByDefault(String resp, List preFields, HttpProtocol http, + CollectRep.MetricsData.Builder builder) { + + } + + /** + * 创建httpContext + * @param httpProtocol http protocol + * @return context + */ + private HttpContext createHttpContext(HttpProtocol httpProtocol) { + HttpProtocol.Authorization auth = httpProtocol.getAuthorization(); + if (auth != null && !DispatchConstants.BEARER_TOKEN.equals(auth.getType())) { + HttpClientContext clientContext = new HttpClientContext(); + if (DispatchConstants.BASIC_AUTH.equals(auth.getType())) { + CredentialsProvider provider = new BasicCredentialsProvider(); + UsernamePasswordCredentials credentials + = new UsernamePasswordCredentials(auth.getBasicAuthUsername(), auth.getBasicAuthPassword()); + provider.setCredentials(AuthScope.ANY, credentials); + clientContext.setCredentialsProvider(provider); + } else if (DispatchConstants.DIGEST_AUTH.equals(auth.getType())) { + CredentialsProvider provider = new BasicCredentialsProvider(); + UsernamePasswordCredentials credentials + = new UsernamePasswordCredentials(auth.getBasicAuthUsername(), auth.getBasicAuthPassword()); + provider.setCredentials(AuthScope.ANY, credentials); + clientContext.setCredentialsProvider(provider); + } else { + clientContext = null; + } + return clientContext; + } + return null; + } + + /** + * 根据http配置参数构造请求头 + * @param httpProtocol http参数配置 + * @return 请求体 + */ + private HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) { + RequestBuilder requestBuilder; + // method + if (HttpMethod.GET.matches(httpProtocol.getMethod())) { + requestBuilder = RequestBuilder.get(); + } else if (HttpMethod.POST.matches(httpProtocol.getMethod())) { + requestBuilder = RequestBuilder.post(); + } else if (HttpMethod.PUT.matches(httpProtocol.getMethod())) { + requestBuilder = RequestBuilder.put(); + } else if (HttpMethod.DELETE.matches(httpProtocol.getMethod())) { + requestBuilder = RequestBuilder.delete(); + } else if (HttpMethod.PATCH.matches(httpProtocol.getMethod())) { + requestBuilder = RequestBuilder.patch(); + } else { + // not support the method + log.error("not support the http method: {}.", httpProtocol.getMethod()); + return null; + } + // params + Map params = httpProtocol.getParams(); + if (params != null && !params.isEmpty()) { + for (Map.Entry param : params.entrySet()) { + requestBuilder.addParameter(param.getKey(), param.getValue()); + } + } + // headers + Map headers = httpProtocol.getHeaders(); + if (headers != null && !headers.isEmpty()) { + for (Map.Entry header : headers.entrySet()) { + requestBuilder.addHeader(header.getKey(), header.getValue()); + } + } + // keep-alive + requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive"); + // add accept + if (DispatchConstants.PARSE_DEFAULT.equals(httpProtocol.getParseType()) + || DispatchConstants.PARSE_JSON_PATH.equals(httpProtocol.getParseType())) { + requestBuilder.addHeader(HttpHeaders.ACCEPT, "application/json"); + } else if (DispatchConstants.PARSE_XML_PATH.equals(httpProtocol.getParseType())) { + requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/xml,application/xml"); + } else if (DispatchConstants.PARSE_PROMETHEUS.equals(httpProtocol.getParseType())) { + requestBuilder.addHeader(HttpHeaders.ACCEPT, DispatchConstants.PARSE_PROMETHEUS_ACCEPT); + requestBuilder.addHeader(HttpHeaders.ACCEPT_ENCODING, "gzip"); + } + + // 判断是否使用Bearer Token认证 + if (httpProtocol.getAuthorization() != null + && DispatchConstants.BEARER_TOKEN.equals(httpProtocol.getAuthorization().getType())) { + // 若使用 将token放入到header里面 + String value = DispatchConstants.BEARER + " " + httpProtocol.getAuthorization().getBearerTokenToken(); + requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value); + } + // todo 处理请求内容 body 暂不支持body + + // uri + requestBuilder.setUri(httpProtocol.getUrl()); + return requestBuilder.build(); + } + + private static class Singleton { + private static final HttpCollectImpl INSTANCE = new HttpCollectImpl(); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/CollectorProperties.java b/collector/server/src/main/java/com/usthe/collector/common/CollectorProperties.java new file mode 100644 index 0000000..9b3cb1d --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/CollectorProperties.java @@ -0,0 +1,15 @@ +package com.usthe.collector.common; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * java common 的配置属性 + * @author tomsun28 + * @date 2021/10/16 14:23 + */ +@Component +@ConfigurationProperties(prefix = "collector.common") +public class CollectorProperties { + +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/cache/CacheCloseable.java b/collector/server/src/main/java/com/usthe/collector/common/cache/CacheCloseable.java new file mode 100644 index 0000000..9960729 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/cache/CacheCloseable.java @@ -0,0 +1,14 @@ +package com.usthe.collector.common.cache; + +/** + * 连接资源关闭回调接口 + * @author tomsun28 + * @date 2021/9/1 21:03 + */ +public interface CacheCloseable { + + /** + * 在缓存remove掉此对象前，回调接口对连接对象进行相关资源的释放 + */ + void close(); +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/cache/CacheDetectable.java b/collector/server/src/main/java/com/usthe/collector/common/cache/CacheDetectable.java new file mode 100644 index 0000000..2fcc814 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/cache/CacheDetectable.java @@ -0,0 +1,15 @@ +package com.usthe.collector.common.cache; + +/** + * 缓存可用性检测接口 + * @author tomsun28 + * @date 2020-08-10 23:21 + */ +public interface CacheDetectable { + + /** + * 可用性探测，探测缓存对象资源的可用性，若不可用，上层可删除此缓存对象 + * @return true 可用, false 不可用 + */ + boolean available(); +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/cache/CacheIdentifier.java b/collector/server/src/main/java/com/usthe/collector/common/cache/CacheIdentifier.java new file mode 100644 index 0000000..9147714 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/cache/CacheIdentifier.java @@ -0,0 +1,23 @@ +package com.usthe.collector.common.cache; + +import lombok.Builder; +import lombok.Data; + +/** + * 缓存key唯一标识符 + * @author tomsun28 + * @date 2021/9/1 21:30 + */ +@Data +@Builder +public class CacheIdentifier { + + private String ip; + + private String port; + + private String username; + + private String password; + +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/cache/CommonCache.java b/collector/server/src/main/java/com/usthe/collector/common/cache/CommonCache.java new file mode 100644 index 0000000..75a1077 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/cache/CommonCache.java @@ -0,0 +1,218 @@ +package com.usthe.collector.common.cache; + + +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * lru cache 对连接对象进行缓存 + * @author tomsun28 + * @date 2020-08-10 23:17 + */ +@Slf4j +public class CommonCache { + + /** + * 默认缓存时间 30minute + */ + private static final long DEFAULT_CACHE_TIMEOUT = 30 * 60 * 1000L; + + /** + * 默认最大缓存数量 + */ + private static final int DEFAULT_MAX_CAPACITY = 10000; + + /** + * cacheTime数组大小 + */ + private static final int CACHE_TIME_LENGTH = 2; + + /** + * 存储对象的数据过期时间点 + */ + private Map timeoutMap; + + /** + * 存储缓存对象 + */ + private ConcurrentLinkedHashMap cacheMap; + + /** + * 过期数据清理线程池 + */ + private ThreadPoolExecutor cleanTimeoutExecutor; + + private CommonCache() { init();} + + /** + * 初始化 cache + */ + private void init() { + // 初始化lru hashmap + cacheMap = new ConcurrentLinkedHashMap + .Builder<>() + .maximumWeightedCapacity(DEFAULT_MAX_CAPACITY) + .listener((key, value) -> { + timeoutMap.remove(key); + if (value instanceof CacheCloseable) { + ((CacheCloseable)value).close(); + } + log.info("lru cache discard key: {}, value: {}.", key, value); + }).build(); + + // 初始化时间纪录map + timeoutMap = new ConcurrentHashMap<>(DEFAULT_MAX_CAPACITY >> 6); + + // 初始化过期数据清理线程池 + cleanTimeoutExecutor = new ThreadPoolExecutor(1, 1, + 1, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(1), r -> new Thread("lru-cache-timeout-cleaner"), + new ThreadPoolExecutor.DiscardOldestPolicy()); + + // 初始化可用性探测定位任务,每次探测间隔时间为20分钟 + ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, + r -> new Thread(r, "lru-cache-available-detector")); + scheduledExecutor.scheduleWithFixedDelay(this::detectCacheAvailable, + 2,20, TimeUnit.MINUTES); + } + + /** + * 探测所有可探测的缓存对象的可用性，清除不可用和过期对象 + */ + private void detectCacheAvailable() { + try { + cacheMap.forEach((key, value) -> { + // 先判断是否过期 + Long[] cacheTime = timeoutMap.get(key); + long currentTime = System.currentTimeMillis(); + if (cacheTime == null || cacheTime.length != CACHE_TIME_LENGTH + || cacheTime[0] + cacheTime[1] < currentTime) { + cacheMap.remove(key); + timeoutMap.remove(key); + if (value instanceof CacheCloseable) { + ((CacheCloseable)value).close(); + } + + } // 对实现了CacheDetectable 的对象探测检查 + else if (value instanceof CacheDetectable && !((CacheDetectable)value).available()) { + cacheMap.remove(key); + timeoutMap.remove(key); + if (value instanceof CacheCloseable) { + ((CacheCloseable)value).close(); + } + } + }); + } catch (Exception e) { + log.error("detect cache available error: {}.", e.getMessage(), e); + } + } + + /** + * 清理过期线程 + */ + private void cleanTimeoutCache() { + try { + cacheMap.forEach((key, value) -> { + // index 0 is startTime, 1 is timeDiff + Long[] cacheTime = timeoutMap.get(key); + long currentTime = System.currentTimeMillis(); + if (cacheTime == null || cacheTime.length != CACHE_TIME_LENGTH) { + timeoutMap.put(key, new Long[]{currentTime, DEFAULT_CACHE_TIMEOUT}); + } else if (cacheTime[0] + cacheTime[1] < currentTime) { + // 过期了 discard 关闭这个cache的资源 + timeoutMap.remove(key); + cacheMap.remove(key); + if (value instanceof CacheCloseable) { + ((CacheCloseable)value).close(); + } + } + }); + } catch (Exception e) { + log.error("clean timeout cache error: {}.", e.getMessage(), e); + } + } + + /** + * 新增或更新cache + * @param key 存储对象key + * @param value 存储对象 + * @param timeDiff 缓存对象保存时间 millis + */ + public void addCache(Object key, Object value, Long timeDiff) { + if (timeDiff == null) { + timeDiff = DEFAULT_CACHE_TIMEOUT; + } + cacheMap.put(key, value); + timeoutMap.put(key, new Long[]{System.currentTimeMillis(), timeDiff}); + cleanTimeoutExecutor.execute(() -> { + try { + cleanTimeoutCache(); + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + log.error(e.getMessage(), e); + } + }); + } + + /** + * 根据缓存key获取缓存对象 + * @param key key + * @param refreshCache 是否刷新命中的缓存的存活时间 true是,false否 + * @return 缓存对象 + */ + public Optional getCache(Object key, boolean refreshCache) { + Long[] cacheTime = timeoutMap.get(key); + if (cacheTime == null || cacheTime.length != CACHE_TIME_LENGTH) { + return Optional.empty(); + } + if (cacheTime[0] + cacheTime[1] < System.currentTimeMillis()) { + timeoutMap.remove(key); + cacheMap.remove(key); + return Optional.empty(); + } + Object value = cacheMap.get(key); + if (value == null) { + cacheMap.remove(key); + timeoutMap.remove(key); + } else if (refreshCache) { + cacheTime[0] = System.currentTimeMillis(); + timeoutMap.put(key, cacheTime); + } + return Optional.ofNullable(value); + } + + /** + * 根据缓存key删除缓存对象 + * @param key key + */ + public void removeCache(Object key) { + timeoutMap.remove(key); + cacheMap.remove(key); + } + + /** + * 获取缓存实例 + * @return cache + */ + public static CommonCache getInstance() { + return SingleInstance.INSTANCE; + } + + /** + * 静态内部类 + */ + private static class SingleInstance { + /** + * 单例 + */ + private static final CommonCache INSTANCE= new CommonCache(); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/cache/support/CommonJdbcConnect.java b/collector/server/src/main/java/com/usthe/collector/common/cache/support/CommonJdbcConnect.java new file mode 100644 index 0000000..6a13c0d --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/cache/support/CommonJdbcConnect.java @@ -0,0 +1,55 @@ +package com.usthe.collector.common.cache.support; + +import com.usthe.collector.common.cache.CacheCloseable; +import com.usthe.collector.common.cache.CacheDetectable; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; + +/** + * @author tomsun28 + * @date 2021/9/1 21:24 + */ +@Slf4j +public class CommonJdbcConnect implements CacheCloseable, CacheDetectable { + + private Connection connection; + + public CommonJdbcConnect(Connection connection) { + this.connection = connection; + } + + @Override + public void close() { + try { + if (connection != null) { + connection.close(); + } + } catch (Exception e) { + log.error("close jdbc connect error: {}", e.getMessage()); + } + } + + @Override + public boolean available() { + if (connection == null) { + return false; + } + try { + return !connection.isClosed(); + } catch (Exception e) { + log.error("detect jdbc connect error: {}", e.getMessage()); + return false; + } + } + + @Override + protected void finalize() throws Throwable { + close(); + super.finalize(); + } + + public Connection getConnection() { + return connection; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/http/CustomHttpRequestRetryHandler.java b/collector/server/src/main/java/com/usthe/collector/common/http/CustomHttpRequestRetryHandler.java new file mode 100644 index 0000000..3d66219 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/http/CustomHttpRequestRetryHandler.java @@ -0,0 +1,116 @@ +package com.usthe.collector.common.http; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequest; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.impl.client.RequestWrapper; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.Args; + +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * http request网络请求重试处理器,替换默认的 DefaultHttpRequestRetryHandler + * 判断是否需要重试请求 + * @author tomsun28 + * @date 2021/8/30 22:00 + */ +@Slf4j +public class CustomHttpRequestRetryHandler implements HttpRequestRetryHandler { + + /** + * 不需要重试的异常情况 + */ + private final static Set> NON_RETRY_CLASSES = new HashSet<>(Arrays.asList( + // io 中断 + InterruptedIOException.class, + // 对端不可达 + UnknownHostException.class, + // 连接异常 + ConnectException.class, + // ssl异常 + SSLException.class)); + /** + * 最大重试次数 + */ + private static final int MAX_RETRY_TIMES = 3; + + @Override + public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { + Args.notNull(exception, "Exception parameter"); + Args.notNull(context, "HTTP context"); + if (executionCount > MAX_RETRY_TIMES) { + return false; + } + if (NON_RETRY_CLASSES.contains(exception.getClass())) { + return false; + } + for (final Class rejectException : NON_RETRY_CLASSES) { + if (rejectException.isInstance(exception)) { + return false; + } + } + // SocketTimeoutException: Read timed 只做一次重试 + // read time out 可能是对端不支持长连接 重试前清理边空闲无效连接 + if (executionCount == 1 && exception instanceof SocketTimeoutException && exception.getMessage() != null + && (exception.getMessage().contains("Read timed out") + || exception.getMessage().contains("Request already aborted"))) { + // 清理连接池无效空闲连接 + HttpClientPool.getConnectionManager().closeExpiredConnections(); + return true; + } + + final HttpClientContext clientContext = HttpClientContext.adapt(context); + final HttpRequest request = clientContext.getRequest(); + + if(requestIsAborted(request)){ + return false; + } + + if (handleAsIdempotent(request)) { + // Retry if the request is considered idempotent + return true; + } + + if (!clientContext.isRequestSent()) { + // Retry if the request has not been sent fully or + // if it's OK to retry methods that have been sent + return true; + } + // otherwise do not retry + return false; + } + + /** + * @since 4.2 + */ + protected boolean handleAsIdempotent(final HttpRequest request) { + return !(request instanceof HttpEntityEnclosingRequest); + } + + /** + * @since 4.2 + * + * @deprecated (4.3) + */ + @Deprecated + protected boolean requestIsAborted(final HttpRequest request) { + HttpRequest req = request; + if (request instanceof RequestWrapper) { + // does not forward request to original + req = ((RequestWrapper) request).getOriginal(); + } + return (req instanceof HttpUriRequest && ((HttpUriRequest)req).isAborted()); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/http/HttpClientPool.java b/collector/server/src/main/java/com/usthe/collector/common/http/HttpClientPool.java new file mode 100644 index 0000000..0959c77 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/http/HttpClientPool.java @@ -0,0 +1,170 @@ +package com.usthe.collector.common.http; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.CookieSpecs; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Lookup; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.util.PublicSuffixMatcher; +import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.apache.http.cookie.CookieSpecProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.impl.cookie.DefaultCookieSpecProvider; +import org.apache.http.impl.cookie.IgnoreSpecProvider; +import org.apache.http.impl.cookie.NetscapeDraftSpecProvider; +import org.apache.http.impl.cookie.RFC6265CookieSpecProvider; +import org.apache.http.impl.cookie.RFC6265CookieSpecProvider.CompatibilityLevel; +import org.apache.http.ssl.SSLContexts; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.concurrent.TimeUnit; + +/** + * 统一的http客户端连接池 + * @author tomsun28 + * @date 2021/8/30 21:23 + */ +@Slf4j +public class HttpClientPool { + + private static CloseableHttpClient httpClient; + + private static PoolingHttpClientConnectionManager connectionManager; + + /** + * 此连接池所能提供的最大连接数 + */ + private final static int MAX_TOTAL_CONNECTIONS = 50000; + + /** + * 每个路由所能分配的最大连接数 + */ + private final static int MAX_PER_ROUTE_CONNECTIONS = 80; + + /** + * 从连接池中获取连接的默认超时时间 4秒 + */ + private final static int REQUIRE_CONNECT_TIMEOUT = 4000; + + /** + * 双端建立连接超时时间 4秒 + */ + private final static int CONNECT_TIMEOUT = 4000; + + /** + * socketReadTimeout 响应tcp报文的最大间隔超时时间 + */ + private final static int SOCKET_TIMEOUT = 60000; + + /** + * 空闲连接免检的有效时间，被重用的空闲连接若超过此时间，需检查此连接的可用性 + */ + private final static int INACTIVITY_VALIDATED_TIME = 10000; + + /** + * ssl版本 + */ + private final static String[] SUPPORTED_SSL = {"TLSv1","TLSv1.1","TLSv1.2","SSLv3"}; + + static { + try { + // 初始化ssl上下文 + SSLContext sslContext = SSLContexts.createDefault(); + X509TrustManager x509TrustManager = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { } + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { } + @Override + public X509Certificate[] getAcceptedIssuers() { return null; } + }; + sslContext.init(null, new TrustManager[]{x509TrustManager}, null); + // 设置支持的ssl版本 + SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, SUPPORTED_SSL, null, new NoopHostnameVerifier()); + // 注册 http https + Registry registry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", sslFactory) + .build(); + // 网络请求默认配置 + RequestConfig requestConfig = RequestConfig.custom() + // 从连接池获取连接超时时间 + .setConnectionRequestTimeout(REQUIRE_CONNECT_TIMEOUT) + // 和对端新连接建立时间，三次握手时间 + .setConnectTimeout(CONNECT_TIMEOUT) + // 数据传输最大响应间隔时间 + .setSocketTimeout(SOCKET_TIMEOUT) + // 遇到301 302不自动重定向跳转 + .setRedirectsEnabled(false) + .build(); + // 连接池 + connectionManager = new PoolingHttpClientConnectionManager(registry); + connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS); + connectionManager.setDefaultMaxPerRoute(MAX_PER_ROUTE_CONNECTIONS); + connectionManager.setValidateAfterInactivity(INACTIVITY_VALIDATED_TIME); + // 请求网络异常重试处理器 + HttpRequestRetryHandler requestRetryHandler = new CustomHttpRequestRetryHandler(); + // cookie处理策略 + PublicSuffixMatcher suffixMatcher = PublicSuffixMatcherLoader.getDefault(); + CookieSpecProvider defaultCookieSpecProvider = new DefaultCookieSpecProvider(suffixMatcher); + CookieSpecProvider laxStandardProvider = new RFC6265CookieSpecProvider(CompatibilityLevel.RELAXED, suffixMatcher); + CookieSpecProvider strictStandardProvider = new RFC6265CookieSpecProvider(CompatibilityLevel.STRICT, suffixMatcher); + Lookup cookieSpecProviderLookup = RegistryBuilder.create() + .register(CookieSpecs.DEFAULT, defaultCookieSpecProvider) + .register("best-match", defaultCookieSpecProvider) + .register("compatibility", defaultCookieSpecProvider) + .register(CookieSpecs.STANDARD, laxStandardProvider) + .register(CookieSpecs.STANDARD_STRICT, strictStandardProvider) + .register(CookieSpecs.NETSCAPE, new NetscapeDraftSpecProvider()) + .register(CookieSpecs.IGNORE_COOKIES, new IgnoreSpecProvider()) + .register(IgnoreReqCookieSpec.COOKIE_SPEC_NAME, new IgnoreReqCookieSpecProvider(suffixMatcher)) + .build(); + // 构造单例 httpClient + httpClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .setDefaultRequestConfig(requestConfig) + .setRetryHandler(requestRetryHandler) + .setDefaultCookieSpecRegistry(cookieSpecProviderLookup) + // 定期清理不可用过期连接 + .evictExpiredConnections() + // 定期清理可用但空闲的连接 + .evictIdleConnections(100, TimeUnit.SECONDS) + .build(); + // 构造连接清理器 + Thread connectCleaner = new Thread(() -> { + while (Thread.currentThread().isInterrupted()) { + try { + Thread.sleep(30000); + connectionManager.closeExpiredConnections(); + connectionManager.closeIdleConnections(100, TimeUnit.SECONDS); + } catch (InterruptedException e) { + } + } + }); + connectCleaner.setName("HttpConnectCleaner"); + connectCleaner.setDaemon(true); + connectCleaner.start(); + } catch (Exception e) { + } + } + + public static CloseableHttpClient getHttpClient() { + return httpClient; + } + + public static PoolingHttpClientConnectionManager getConnectionManager() { + return connectionManager; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpec.java b/collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpec.java new file mode 100644 index 0000000..9b0f60d --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpec.java @@ -0,0 +1,29 @@ +package com.usthe.collector.common.http; + +import org.apache.http.cookie.Cookie; +import org.apache.http.cookie.CookieOrigin; +import org.apache.http.impl.cookie.DefaultCookieSpec; + +/** + * 在request请求时忽略携带cookie,返回response时更新cookie的策略 + * @author tomsun28 + * @date 2021/8/30 22:37 + */ +public class IgnoreReqCookieSpec extends DefaultCookieSpec { + + public static final String COOKIE_SPEC_NAME = "IGNORE_REQUEST_COOKIE"; + + public IgnoreReqCookieSpec() { + super(); + } + + @Override + public boolean match(Cookie cookie, CookieOrigin origin) { + return false; + } + + @Override + public String toString() { + return COOKIE_SPEC_NAME; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpecProvider.java b/collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpecProvider.java new file mode 100644 index 0000000..c8c9212 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/common/http/IgnoreReqCookieSpecProvider.java @@ -0,0 +1,32 @@ +package com.usthe.collector.common.http; + +import org.apache.http.conn.util.PublicSuffixMatcher; +import org.apache.http.cookie.CookieSpec; +import org.apache.http.impl.cookie.DefaultCookieSpecProvider; +import org.apache.http.protocol.HttpContext; + +/** + * IGNORE_REQUEST_COOKIE cookie策略提供者 + * @author tomsun28 + * @date 2021/8/30 22:40 + */ +public class IgnoreReqCookieSpecProvider extends DefaultCookieSpecProvider { + + private volatile CookieSpec cookieSpec; + + public IgnoreReqCookieSpecProvider(PublicSuffixMatcher publicSuffixMatcher) { + super(publicSuffixMatcher); + } + + @Override + public CookieSpec create(HttpContext context) { + if (cookieSpec == null) { + synchronized (this) { + if (cookieSpec == null) { + this.cookieSpec = new IgnoreReqCookieSpec(); + } + } + } + return this.cookieSpec; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/CollectDataDispatch.java b/collector/server/src/main/java/com/usthe/collector/dispatch/CollectDataDispatch.java new file mode 100644 index 0000000..a214a40 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/CollectDataDispatch.java @@ -0,0 +1,23 @@ +package com.usthe.collector.dispatch; + + +import com.usthe.collector.dispatch.timer.WheelTimerJob; +import com.usthe.common.entity.job.Metrics; +import com.usthe.common.entity.message.CollectRep; + +/** + * 采集数据调度器接口 + * @author tomsun28 + * @date 2021/11/2 11:20 + */ +public interface CollectDataDispatch { + + /** + * 处理分发采集结果数据 + * @param timerJob 时间轮任务 + * @param metrics 下面的指标组采集任务 + * @param metricsData 采集结果数据 + */ + void dispatchCollectData(WheelTimerJob timerJob, Metrics metrics, CollectRep.MetricsData metricsData); + +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/CommonDispatcher.java b/collector/server/src/main/java/com/usthe/collector/dispatch/CommonDispatcher.java new file mode 100644 index 0000000..a1f7285 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/CommonDispatcher.java @@ -0,0 +1,187 @@ +package com.usthe.collector.dispatch; + +import com.usthe.collector.dispatch.export.KafkaDataExporter; +import com.usthe.collector.dispatch.timer.TimerDispatch; +import com.usthe.collector.dispatch.timer.WheelTimerJob; +import com.usthe.common.entity.job.Job; +import com.usthe.common.entity.job.Metrics; +import com.usthe.common.entity.message.CollectRep; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * 指标组采集任务与响应数据调度器 + * @author tomsun28 + * @date 2021/11/2 11:24 + */ +@Component +@Slf4j +public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatch { + + /** + * 指标组采集任务超时时间值 + */ + private static final long DURATION_TIME = 120_000L; + /** + * 指标组采集任务优先级队列 + */ + private MetricsCollectorQueue jobRequestQueue; + /** + * 时间轮任务调度器 + */ + private TimerDispatch timerDispatch; + /** + * kafka采集数据导出器 + */ + private KafkaDataExporter kafkaDataExporter; + /** + * 指标组任务与开始时间映射map + */ + private Map metricsTimeoutMonitorMap; + + public CommonDispatcher(MetricsCollectorQueue jobRequestQueue, TimerDispatch timerDispatch, + KafkaDataExporter kafkaDataExporter, WorkerPool workerPool) { + this.kafkaDataExporter = kafkaDataExporter; + this.jobRequestQueue = jobRequestQueue; + this.timerDispatch = timerDispatch; + ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(2, 2, 1, + TimeUnit.SECONDS, + new SynchronousQueue<>(), r -> { + Thread thread = new Thread(r); + thread.setDaemon(true); + return thread; + }); + // 从任务队列拉取指标组采集任务放入线程池执行 + poolExecutor.execute(() -> { + Thread.currentThread().setName("metrics-task-dispatcher"); + while (!Thread.currentThread().isInterrupted()) { + MetricsCollect metricsCollect = null; + try { + metricsCollect = jobRequestQueue.getJob(); + workerPool.executeJob(metricsCollect); + } catch (RejectedExecutionException rejected) { + log.info("[Dispatcher]-the worker pool is full, reject this metrics task, " + + "sleep and put in queue again."); + try { + Thread.sleep(1000); + if (metricsCollect != null) { + // 在队列里的优先级增大 + metricsCollect.setRunPriority((byte) (metricsCollect.getRunPriority() + 1)); + jobRequestQueue.addJob(metricsCollect); + } + } catch (InterruptedException interruptedException){} + } catch (Exception e) { + log.error("[Dispatcher]-{}.", e.getMessage(), e); + } + } + }); + // 监控指标组采集任务执行时间 + metricsTimeoutMonitorMap = new ConcurrentHashMap<>(128); + poolExecutor.execute(() -> { + Thread.currentThread().setName("metrics-task-monitor"); + while (!Thread.currentThread().isInterrupted()) { + try { + // 检测每个指标组采集单元是否超时2分钟,超时则丢弃并返回异常 + long deadline = System.currentTimeMillis() - DURATION_TIME; + for (Map.Entry entry : metricsTimeoutMonitorMap.entrySet()) { + MetricsTime metricsTime = entry.getValue(); + if (metricsTime.getStartTime() < deadline) { + // 指标组采集超时 + CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder() + .setId(metricsTime.getTimerJob().getJob().getAppId()) + .setApp(metricsTime.getTimerJob().getJob().getApp()) + .setMetrics(metricsTime.getMetrics().getName()) + .setTime(System.currentTimeMillis()) + .setCode(CollectRep.Code.TIMEOUT).setMsg("collect timeout").build(); + dispatchCollectData(metricsTime.getTimerJob(), metricsTime.getMetrics(), metricsData); + } + } + Thread.sleep(20000); + } catch (Exception e){ + log.error("[Monitor]-{}.", e.getMessage(), e); + } + } + }); + } + + @Override + public void dispatchMetricsTask(WheelTimerJob timerJob) { + // 将单个应用的采集任务根据其下的指标组拆分为对应的指标组采集任务 AbstractCollect + // 将每个指标组放入线程池进行调度 + Job job = timerJob.getJob(); + job.constructMetrics(); + Set metricsSet = job.getNextCollectMetrics(null, true); + metricsSet.forEach(metrics -> { + MetricsCollect metricsCollect = new MetricsCollect(metrics, timerJob, this); + jobRequestQueue.addJob(metricsCollect); + metricsTimeoutMonitorMap.put(job.getId() + metrics.getName(), + new MetricsTime(System.currentTimeMillis(), metrics, timerJob)); + }); + } + + @Override + public void dispatchCollectData(WheelTimerJob timerJob, Metrics metrics, CollectRep.MetricsData metricsData) { + Job job = timerJob.getJob(); + Set metricsSet = job.getNextCollectMetrics(metrics, false); + if (job.isCyclic()) { + // 若是异步的周期性循环任务,直接发送指标组的采集数据到消息中间件 + kafkaDataExporter.send(metricsData); + if (metricsSet == null) { + // 此Job所有指标组采集执行完成 + // 周期性任务再次将任务push到时间轮 + // 先判断此次任务执行时间与任务采集间隔时间 + long spendTime = System.currentTimeMillis() - job.getTimestamp(); + long interval = job.getInterval() - spendTime / 1000; + interval = interval <= 0 ? 0 : interval; + timerDispatch.cyclicJob(timerJob, interval, TimeUnit.SECONDS); + } else if (!metricsSet.isEmpty()) { + // 当前级别指标组执行完成，开始执行下一级别的指标组 + metricsSet.forEach(metricItem -> { + MetricsCollect metricsCollect = new MetricsCollect(metricItem, timerJob, this); + jobRequestQueue.addJob(metricsCollect); + metricsTimeoutMonitorMap.put(job.getId() + metrics.getName(), + new MetricsTime(System.currentTimeMillis(), metrics, timerJob)); + }); + } else { + // 当前执行级别的指标组列表未全执行完成, + // 需等待其它同级别指标组执行完成后进入下一级别执行 + } + } else { + // 若是临时性一次任务,需等待所有指标组的采集数据统一包装返回 + // todo 将当前指标组数据插入job里统一组装 + if (metricsSet == null) { + // 此Job所有指标组采集执行完成 + // todo 将所有指标组数据组合一起发送到回调函数 + } else if (!metricsSet.isEmpty()) { + // 当前级别指标组执行完成，开始执行下一级别的指标组 + metricsSet.forEach(metricItem -> { + MetricsCollect metricsCollect = new MetricsCollect(metricItem, timerJob, this); + jobRequestQueue.addJob(metricsCollect); + metricsTimeoutMonitorMap.put(job.getId() + metrics.getName(), + new MetricsTime(System.currentTimeMillis(), metrics, timerJob)); + }); + } else { + // 当前执行级别的指标组列表未全执行完成, + // 需等待其它同级别指标组执行完成后进入下一级别执行 + } + } + } + + @Data + @AllArgsConstructor + private static class MetricsTime { + private long startTime; + private Metrics metrics; + private WheelTimerJob timerJob; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConfiguration.java b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConfiguration.java new file mode 100644 index 0000000..73f68a8 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConfiguration.java @@ -0,0 +1,21 @@ +package com.usthe.collector.dispatch; + +import com.googlecode.aviator.AviatorEvaluator; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author tomsun28 + * @date 2021/11/3 12:55 + */ +@Configuration +public class DispatchConfiguration { + + private static final int AVIATOR_LRU_CACHE_SIZE = 1024; + + @Bean + public void configAviatorEvaluator() { + // 配置AviatorEvaluator使用LRU缓存编译后的表达式 + AviatorEvaluator.getInstance().useLRUExpressionCache(AVIATOR_LRU_CACHE_SIZE); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java new file mode 100644 index 0000000..6979212 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchConstants.java @@ -0,0 +1,65 @@ +package com.usthe.collector.dispatch; + +/** + * dispatch 常量 + * @author tomsun28 + * @date 2021/11/3 16:50 + */ +public interface DispatchConstants { + + /** + * 可用性指标组 + */ + String AVAILABILITY = "availability"; + + // 协议类型相关 - start // + /** + * 协议 http + */ + String PROTOCOL_HTTP = "http"; + /** + * 协议 icmp + */ + String PROTOCOL_ICMP = "icmp"; + /** + * 协议 jdbc + */ + String PROTOCOL_JDBC = "jdbc"; + // 协议类型相关 - end // + + // http协议相关 - start 需尽可能先复用 HttpHeaders // + /** + * 认证方式 Bearer Token + */ + String BEARER_TOKEN = "Bearer Token"; + /** + * Bearer Token 的认证参数字符 + */ + String BEARER = "Bearer"; + /** + * 认证方式 Basic Auth + */ + String BASIC_AUTH = "Basic Auth"; + /** + * 认证方式 Digest Auth + */ + String DIGEST_AUTH = "Digest Auth"; + /** + * 解析方式 默认规则 + */ + String PARSE_DEFAULT = "default"; + /** + * 解析方式 自定义json path + */ + String PARSE_JSON_PATH = "json_path"; + /** + * 解析方式 自定义xml path + */ + String PARSE_XML_PATH = "xml_path"; + /** + * 解析方式 prometheus规则 + */ + String PARSE_PROMETHEUS = "prometheus"; + String PARSE_PROMETHEUS_ACCEPT = "application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1"; + // http协议相关 - end // +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchProperties.java b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchProperties.java new file mode 100644 index 0000000..c0acde2 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/DispatchProperties.java @@ -0,0 +1,225 @@ +package com.usthe.collector.dispatch; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * 调度分发任务配置属性 + * @author tomsun28 + * @date 2021/10/16 14:54 + */ +@Component +@ConfigurationProperties(prefix = "collector.dispatch") +public class DispatchProperties { + + /** + * 调度入口配置属性 + */ + private EntranceProperties entrance; + + /** + * 调度数据出口配置属性 + */ + private ExportProperties export; + + public EntranceProperties getEntrance() { + return entrance; + } + + public void setEntrance(EntranceProperties entrance) { + this.entrance = entrance; + } + + public ExportProperties getExport() { + return export; + } + + public void setExport(ExportProperties export) { + this.export = export; + } + + /** + * 调度入口配置属性 + * 入口可以时etcd信息,http请求,消息中间件消息请求 + */ + public static class EntranceProperties { + /** + * etcd配置信息 + */ + private EtcdProperties etcd; + + public EtcdProperties getEtcd() { + return etcd; + } + + public void setEtcd(EtcdProperties etcd) { + this.etcd = etcd; + } + + public static class EtcdProperties { + + /** + * etcd调度是否启动 + */ + private boolean enabled = true; + + /** + * etcd的连接端点url + */ + private String[] endpoints = new String[] {"http://127.0.0.1:2379"}; + + /** + * etcd连接用户名 + */ + private String username; + + /** + * etcd连接密码 + */ + private String password; + + /** + * etcd租约的有效时间 单位秒 + */ + private long ttl = 200; + + /** + * 采集器注册目录 + */ + private String collectorDir = "/usthe/dispatch/collector/"; + + /** + * 任务调度分发目录 + */ + private String assignDir = "/usthe/dispatch/assign/"; + + /** + * 任务详细目录 + */ + private String jobDir = "/usthe/dispatch/job/"; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String[] getEndpoints() { + return endpoints; + } + + public void setEndpoints(String[] endpoints) { + this.endpoints = endpoints; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public long getTtl() { + return ttl; + } + + public void setTtl(long ttl) { + this.ttl = ttl; + } + + public String getCollectorDir() { + return collectorDir; + } + + public void setCollectorDir(String collectorDir) { + this.collectorDir = collectorDir; + } + + public String getAssignDir() { + return assignDir; + } + + public void setAssignDir(String assignDir) { + this.assignDir = assignDir; + } + + public String getJobDir() { + return jobDir; + } + + public void setJobDir(String jobDir) { + this.jobDir = jobDir; + } + } + } + + /** + * 调度数据出口配置属性 + */ + public static class ExportProperties { + + /** + * kafka配置信息 + */ + private KafkaProperties kafka; + + public KafkaProperties getKafka() { + return kafka; + } + + public void setKafka(KafkaProperties kafka) { + this.kafka = kafka; + } + + public static class KafkaProperties { + /** + * kafka数据出口是否启动 + */ + private boolean enabled = true; + + /** + * kafka的连接服务器url + */ + private String servers = "http://127.0.0.1:2379"; + /** + * 发送数据的topic名称 + */ + private String topic; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getServers() { + return servers; + } + + public void setServers(String servers) { + this.servers = servers; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + } + } +} \ No newline at end of file diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java b/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java new file mode 100644 index 0000000..9343c83 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java @@ -0,0 +1,166 @@ +package com.usthe.collector.dispatch; + +import com.google.protobuf.ProtocolStringList; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.Expression; +import com.usthe.collector.collect.AbstractCollect; +import com.usthe.collector.collect.http.HttpCollectImpl; +import com.usthe.collector.dispatch.timer.WheelTimerJob; +import com.usthe.common.entity.job.Metrics; +import com.usthe.common.entity.message.CollectRep; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * parent job + * @author tomsun28 + * @date 2021/10/10 15:35 + */ +@Slf4j +@Data +public class MetricsCollect implements Runnable, Comparable { + + protected byte runPriority; + + protected long newTime; + + protected long startTime; + + protected Metrics metrics; + + protected WheelTimerJob timerJob; + + protected CollectDataDispatch collectDataDispatch; + + public MetricsCollect(Metrics metrics, WheelTimerJob timerJob, CollectDataDispatch collectDataDispatch) { + this.newTime = System.currentTimeMillis(); + this.timerJob = timerJob; + this.metrics = metrics; + this.collectDataDispatch = collectDataDispatch; + if (DispatchConstants.AVAILABILITY.equals(metrics.getName())) { + runPriority = (byte) 1; + } else { + runPriority = (byte) -1; + } + } + + @Override + public void run() { + this.startTime = System.currentTimeMillis(); + setNewThreadName(timerJob, metrics); + CollectRep.MetricsData.Builder response = CollectRep.MetricsData.newBuilder(); + response.setApp(timerJob.getJob().getApp()); + response.setId(timerJob.getJob().getId()); + response.setMetrics(metrics.getName()); + + // 根据指标组采集协议,应用类型等来调度到真正的应用指标组采集实现类 + AbstractCollect abstractCollect = null; + switch (metrics.getProtocol()) { + case DispatchConstants.PROTOCOL_HTTP: + abstractCollect = HttpCollectImpl.getInstance(); + break; + // todo + default: break; + } + if (abstractCollect == null) { + log.error("[Dispatcher] - not support this: app: {}, metrics: {}, protocol: {}.", + timerJob.getJob().getApp(), metrics.getName(), metrics.getProtocol()); + response.setCode(CollectRep.Code.FAIL); + response.setMsg("not support " + timerJob.getJob().getApp() + ", " + + metrics.getName() + ", " + metrics.getProtocol()); + return; + } else { + try { + abstractCollect.collect(response, timerJob.getJob().getAppId(), + timerJob.getJob().getApp(), metrics); + } catch (Exception e) { + log.error("[Metrics Collect]: {}.", e.getMessage(), e); + response.setCode(CollectRep.Code.FAIL); + response.setMsg(e.getMessage()); + } + } + // 别名属性表达式替换计算 + calculateFields(metrics, response); + CollectRep.MetricsData metricsData = validateResponse(response); + collectDataDispatch.dispatchCollectData(timerJob, metrics, metricsData); + } + + /** + * 根据 calculates 和 aliasFields 配置计算出真正的指标(fields)值 + * @param metrics 指标组配置 + * @param collectData 采集数据 + */ + private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder collectData) { + collectData.addAllFields(metrics.getFields()); + // 若不存在需要计算的表达式,则 别名指标aliasFields 的数据就是真正指标 fields的数据 + if (metrics.getCalculates() == null || metrics.getCalculates().isEmpty()) { + return; + } + List aliasRowList = collectData.getValuesList(); + if (aliasRowList == null || aliasRowList.isEmpty()) { + return; + } + // 先预处理 calculates + Map fieldExpressionMap = metrics.getCalculates() + .stream() + .map(cal -> { + int splitIndex = cal.indexOf("="); + String field = cal.substring(0, splitIndex); + String expressionStr = cal.substring(splitIndex + 1); + Expression expression = AviatorEvaluator.compile(expressionStr, true); + return new Object[]{field, expression}; }) + .collect(Collectors.toMap(arr -> (String)arr[0], arr -> (Expression) arr[1])); + + List fields = metrics.getFields(); + List aliasFields = metrics.getAliasFields(); + Map aliasFieldValueMap = new HashMap<>(16); + for (int index = 0; index < aliasRowList.size(); index++) { + CollectRep.ValueRow aliasRow = aliasRowList.get(index); + for (int aliasIndex = 0; aliasIndex < aliasFields.size(); aliasIndex++) { + aliasFieldValueMap.put(aliasFields.get(aliasIndex), aliasRow.getColumns(aliasIndex)); + } + ProtocolStringList columnList = aliasRow.getColumnsList(); + columnList.clear(); + for (int realIndex = 0; realIndex < fields.size(); realIndex++) { + String realField = fields.get(realIndex); + Expression expression = fieldExpressionMap.get(realField); + String value = ""; + if (expression != null) { + // 存在计算表达式 则计算值 + value = (String) expression.execute(aliasFieldValueMap); + } else { + // 不存在 则映射别名值 + value = (String) aliasFieldValueMap.get(realField); + } + columnList.add(value); + } + } + } + + private CollectRep.MetricsData validateResponse(CollectRep.MetricsData.Builder builder) { + long endTime = System.currentTimeMillis(); + builder.setTime(endTime); + log.debug("[Collect]: newTime: {}, startTime: {}, spendTime: {}.", newTime, startTime, endTime - startTime); + if (builder.getCode() != CollectRep.Code.SUCCESS) { + log.info("[Collect Fail]-reason:{}", builder.getMsg()); + } else { + log.info("[Collect Success]-{},{},{}.", builder.getId(), builder.getApp(), builder.getMetrics()); + } + return builder.build(); + } + + private void setNewThreadName(WheelTimerJob timerJob, Metrics metrics) { + String currentName = timerJob.getJob().getAppId() + timerJob.getJob().getApp() + metrics.getName() + timerJob.getJob().getId(); + Thread.currentThread().setName(currentName); + } + + @Override + public int compareTo(MetricsCollect collect) { + return runPriority - collect.runPriority; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollectorQueue.java b/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollectorQueue.java new file mode 100644 index 0000000..8275bd9 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsCollectorQueue.java @@ -0,0 +1,32 @@ +package com.usthe.collector.dispatch; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * 待运行的job队列 + * @author tomsun28 + * @date 2021/10/10 20:20 + */ +@Component +@Slf4j +public class MetricsCollectorQueue { + + private final PriorityBlockingQueue jobQueue; + + public MetricsCollectorQueue() { + jobQueue = new PriorityBlockingQueue<>(2000); + } + + public void addJob(MetricsCollect job) { + jobQueue.offer(job); + } + + public MetricsCollect getJob() throws InterruptedException { + return jobQueue.poll(2, TimeUnit.SECONDS); + } + +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsTaskDispatch.java b/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsTaskDispatch.java new file mode 100644 index 0000000..b8d039f --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/MetricsTaskDispatch.java @@ -0,0 +1,17 @@ +package com.usthe.collector.dispatch; + +import com.usthe.collector.dispatch.timer.WheelTimerJob; + +/** + * 指标组采集任务调度器接口 + * @author tomsun28 + * @date 2021/11/2 11:19 + */ +public interface MetricsTaskDispatch { + + /** + * 调度 + * @param timerJob timerJob + */ + void dispatchMetricsTask(WheelTimerJob timerJob); +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/WorkerPool.java b/collector/server/src/main/java/com/usthe/collector/dispatch/WorkerPool.java new file mode 100644 index 0000000..39d4898 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/WorkerPool.java @@ -0,0 +1,54 @@ +package com.usthe.collector.dispatch; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * 采集任务工作线程池 + * @author tomsun28 + * @date 2021/10/15 0:01 + */ +@Component +@Slf4j +public class WorkerPool { + + private ThreadPoolExecutor workerExecutor; + + public WorkerPool() { + initWorkExecutor(); + } + + private void initWorkExecutor() { + // 线程工厂 + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler((thread, throwable) -> { + log.error("workerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); }) + .setDaemon(true) + .setNameFormat("collect-worker-%d") + .build(); + workerExecutor = new ThreadPoolExecutor(100, + 800, + 10, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + threadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + /** + * 运行采集任务线程 + * @param runnable 任务 + * @throws RejectedExecutionException when 线程池满 + */ + public void executeJob(Runnable runnable) throws RejectedExecutionException { + workerExecutor.execute(runnable); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/entrance/http/CollectJobController.java b/collector/server/src/main/java/com/usthe/collector/dispatch/entrance/http/CollectJobController.java new file mode 100644 index 0000000..dc4490d --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/entrance/http/CollectJobController.java @@ -0,0 +1,25 @@ +package com.usthe.collector.dispatch.entrance.http; + +import com.usthe.common.entity.job.Job; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; + +/** + * 采集job管理提供api接口 + * @author tomsun28 + * @date 2021/11/6 13:58 + */ +@RestController +public class CollectJobController { + + /** + * 执行一次性采集任务,获取采集数据响应 + * @return 采集结果 + */ + @PostMapping("/job") + public Mono collectJobData(Job job) { + return null; + } + +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaDataExporter.java b/collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaDataExporter.java new file mode 100644 index 0000000..1bc06b5 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaDataExporter.java @@ -0,0 +1,54 @@ +package com.usthe.collector.dispatch.export; + +import com.usthe.collector.dispatch.DispatchProperties; +import com.usthe.common.entity.message.CollectRep; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; + +import java.util.Properties; + +/** + * kafka采集数据消息发送 + * @author tomsun28 + * @date 2021/11/3 15:22 + */ +@Configuration +@ConditionalOnProperty(prefix = "collector.dispatch.export.kafka", + name = "enabled", havingValue = "true", matchIfMissing = true) +@AutoConfigureAfter(value = {DispatchProperties.class}) +@Slf4j +public class KafkaDataExporter { + + KafkaProducer kafkaProducer; + DispatchProperties.ExportProperties.KafkaProperties kafkaProperties; + public KafkaDataExporter(DispatchProperties dispatchProperties) { + try { + kafkaProperties = dispatchProperties.getExport().getKafka(); + Properties properties = new Properties(); + properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getServers()); + properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaMetricsDataSerializer.class); + kafkaProducer = new KafkaProducer<>(properties); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + + /** + * 发送消息 + * @param metricsData 指标组采集数据 + */ + public void send(CollectRep.MetricsData metricsData) { + if (kafkaProducer != null) { + kafkaProducer.send(new ProducerRecord<>(kafkaProperties.getTopic(), metricsData)); + } else { + log.error("kafkaProducer is not enable"); + } + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaMetricsDataSerializer.java b/collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaMetricsDataSerializer.java new file mode 100644 index 0000000..d21afe7 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/export/KafkaMetricsDataSerializer.java @@ -0,0 +1,18 @@ +package com.usthe.collector.dispatch.export; + + +import com.usthe.common.entity.message.CollectRep; +import org.apache.kafka.common.serialization.Serializer; + +/** + * MetricsData的序列化 + * @author tomsun28 + * @date 2021/11/3 16:14 + */ +public class KafkaMetricsDataSerializer implements Serializer { + + @Override + public byte[] serialize(String topicName, CollectRep.MetricsData metricsData) { + return metricsData.toByteArray(); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/HashedWheelTimer.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/HashedWheelTimer.java new file mode 100644 index 0000000..9e71563 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/HashedWheelTimer.java @@ -0,0 +1,809 @@ +/* + * Copyright 2012 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.usthe.collector.dispatch.timer; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A {@link Timer} optimized for approximated I/O timeout scheduling. + * + *

Tick Duration

+ *

+ * As described with 'approximated', this timer does not execute the scheduled + * {@link TimerTask} on time. {@link HashedWheelTimer}, on every tick, will + * check if there are any {@link TimerTask}s behind the schedule and execute + * them. + *

+ * You can increase or decrease the accuracy of the execution timing by + * specifying smaller or larger tick duration in the constructor. In most + * network applications, I/O timeout does not need to be accurate. Therefore, + * the default tick duration is 100 milliseconds and you will not need to try + * different configurations in most cases. + * + *

Ticks per Wheel (Wheel Size)

+ *

+ * {@link HashedWheelTimer} maintains a data structure called 'wheel'. + * To put simply, a wheel is a hash table of {@link TimerTask}s whose hash + * function is 'dead line of the task'. The default number of ticks per wheel + * (i.e. the size of the wheel) is 512. You could specify a larger value + * if you are going to schedule a lot of timeouts. + * + *

Do not create many instances.

+ *

+ * {@link HashedWheelTimer} creates a new thread whenever it is instantiated and + * started. Therefore, you should make sure to create only one instance and + * share it across your application. One of the common mistakes, that makes + * your application unresponsive, is to create a new instance for every connection. + * + *

Implementation Details

+ *

+ * {@link HashedWheelTimer} is based on + * George Varghese and + * Tony Lauck's paper, + * 'Hashed + * and Hierarchical Timing Wheels: data structures to efficiently implement a + * timer facility'. More comprehensive slides are located + * here. + * @author from netty | dubbo + */ +@SuppressWarnings("PMD") +public class HashedWheelTimer implements Timer { + + private static final Logger logger = LoggerFactory.getLogger(HashedWheelTimer.class); + + private static final AtomicInteger INSTANCE_COUNTER = new AtomicInteger(); + private static final AtomicBoolean WARNED_TOO_MANY_INSTANCES = new AtomicBoolean(); + private static final int INSTANCE_COUNT_LIMIT = 64; + private static final AtomicIntegerFieldUpdater WORKER_STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimer.class, "workerState"); + + private final Worker worker = new Worker(); + private final Thread workerThread; + + private static final int WORKER_STATE_INIT = 0; + private static final int WORKER_STATE_STARTED = 1; + private static final int WORKER_STATE_SHUTDOWN = 2; + + /** + * 0 - init, 1 - started, 2 - shut down + */ + @SuppressWarnings({"unused", "FieldMayBeFinal"}) + private volatile int workerState; + + private final long tickDuration; + private final HashedWheelBucket[] wheel; + private final int mask; + private final CountDownLatch startTimeInitialized = new CountDownLatch(1); + private final Queue timeouts = new LinkedBlockingQueue<>(); + private final Queue cancelledTimeouts = new LinkedBlockingQueue<>(); + private final AtomicLong pendingTimeouts = new AtomicLong(0); + private final long maxPendingTimeouts; + + private volatile long startTime; + + /** + * Creates a new timer with the default thread factory + * ({@link Executors#defaultThreadFactory()}), default tick duration, and + * default number of ticks per wheel. + */ + public HashedWheelTimer() { + this(Executors.defaultThreadFactory()); + } + + /** + * Creates a new timer with the default thread factory + * ({@link Executors#defaultThreadFactory()}) and default number of ticks + * per wheel. + * + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @throws NullPointerException if {@code unit} is {@code null} + * @throws IllegalArgumentException if {@code tickDuration} is <= 0 + */ + public HashedWheelTimer(long tickDuration, TimeUnit unit) { + this(Executors.defaultThreadFactory(), tickDuration, unit); + } + + /** + * Creates a new timer with the default thread factory + * ({@link Executors#defaultThreadFactory()}). + * + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @param ticksPerWheel the size of the wheel + * @throws NullPointerException if {@code unit} is {@code null} + * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 + */ + public HashedWheelTimer(long tickDuration, TimeUnit unit, int ticksPerWheel) { + this(Executors.defaultThreadFactory(), tickDuration, unit, ticksPerWheel); + } + + /** + * Creates a new timer with the default tick duration and default number of + * ticks per wheel. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @throws NullPointerException if {@code threadFactory} is {@code null} + */ + public HashedWheelTimer(ThreadFactory threadFactory) { + this(threadFactory, 100, TimeUnit.MILLISECONDS); + } + + /** + * Creates a new timer with the default number of ticks per wheel. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} + * @throws IllegalArgumentException if {@code tickDuration} is <= 0 + */ + public HashedWheelTimer( + ThreadFactory threadFactory, long tickDuration, TimeUnit unit) { + this(threadFactory, tickDuration, unit, 512); + } + + /** + * Creates a new timer. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @param ticksPerWheel the size of the wheel + * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} + * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 + */ + public HashedWheelTimer( + ThreadFactory threadFactory, + long tickDuration, TimeUnit unit, int ticksPerWheel) { + this(threadFactory, tickDuration, unit, ticksPerWheel, -1); + } + + /** + * Creates a new timer. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @param ticksPerWheel the size of the wheel + * @param maxPendingTimeouts The maximum number of pending timeouts after which call to + * {@code newTimeout} will result in + * {@link RejectedExecutionException} + * being thrown. No maximum pending timeouts limit is assumed if + * this value is 0 or negative. + * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} + * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 + */ + public HashedWheelTimer( + ThreadFactory threadFactory, + long tickDuration, TimeUnit unit, int ticksPerWheel, + long maxPendingTimeouts) { + + if (threadFactory == null) { + throw new NullPointerException("threadFactory"); + } + if (unit == null) { + throw new NullPointerException("unit"); + } + if (tickDuration <= 0) { + throw new IllegalArgumentException("tickDuration must be greater than 0: " + tickDuration); + } + if (ticksPerWheel <= 0) { + throw new IllegalArgumentException("ticksPerWheel must be greater than 0: " + ticksPerWheel); + } + + // Normalize ticksPerWheel to power of two and initialize the wheel. + wheel = createWheel(ticksPerWheel); + mask = wheel.length - 1; + + // Convert tickDuration to nanos. + this.tickDuration = unit.toNanos(tickDuration); + + // Prevent overflow. + if (this.tickDuration >= Long.MAX_VALUE / wheel.length) { + throw new IllegalArgumentException(String.format( + "tickDuration: %d (expected: 0 < tickDuration in nanos < %d", + tickDuration, Long.MAX_VALUE / wheel.length)); + } + workerThread = threadFactory.newThread(worker); + + this.maxPendingTimeouts = maxPendingTimeouts; + + if (INSTANCE_COUNTER.incrementAndGet() > INSTANCE_COUNT_LIMIT && + WARNED_TOO_MANY_INSTANCES.compareAndSet(false, true)) { + reportTooManyInstances(); + } + } + + @Override + protected void finalize() throws Throwable { + try { + super.finalize(); + } finally { + // This object is going to be GCed and it is assumed the ship has sailed to do a proper shutdown. If + // we have not yet shutdown then we want to make sure we decrement the active instance count. + if (WORKER_STATE_UPDATER.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) { + INSTANCE_COUNTER.decrementAndGet(); + } + } + } + + private static HashedWheelBucket[] createWheel(int ticksPerWheel) { + if (ticksPerWheel <= 0) { + throw new IllegalArgumentException( + "ticksPerWheel must be greater than 0: " + ticksPerWheel); + } + if (ticksPerWheel > 1073741824) { + throw new IllegalArgumentException( + "ticksPerWheel may not be greater than 2^30: " + ticksPerWheel); + } + + ticksPerWheel = normalizeTicksPerWheel(ticksPerWheel); + HashedWheelBucket[] wheel = new HashedWheelBucket[ticksPerWheel]; + for (int i = 0; i < wheel.length; i++) { + wheel[i] = new HashedWheelBucket(); + } + return wheel; + } + + private static int normalizeTicksPerWheel(int ticksPerWheel) { + int normalizedTicksPerWheel = ticksPerWheel - 1; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 1; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 2; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 4; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 8; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 16; + return normalizedTicksPerWheel + 1; + } + + /** + * Starts the background thread explicitly. The background thread will + * start automatically on demand even if you did not call this method. + * + * @throws IllegalStateException if this timer has been + * {@linkplain #stop() stopped} already + */ + public void start() { + switch (WORKER_STATE_UPDATER.get(this)) { + case WORKER_STATE_INIT: + if (WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_INIT, WORKER_STATE_STARTED)) { + workerThread.start(); + } + break; + case WORKER_STATE_STARTED: + break; + case WORKER_STATE_SHUTDOWN: + throw new IllegalStateException("cannot be started once stopped"); + default: + throw new Error("Invalid WorkerState"); + } + + // Wait until the startTime is initialized by the worker. + while (startTime == 0) { + try { + startTimeInitialized.await(); + } catch (InterruptedException ignore) { + // Ignore - it will be ready very soon. + } + } + } + + @Override + public Set stop() { + if (Thread.currentThread() == workerThread) { + throw new IllegalStateException( + HashedWheelTimer.class.getSimpleName() + + ".stop() cannot be called from " + + TimerTask.class.getSimpleName()); + } + + if (!WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_STARTED, WORKER_STATE_SHUTDOWN)) { + // workerState can be 0 or 2 at this moment - let it always be 2. + if (WORKER_STATE_UPDATER.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) { + INSTANCE_COUNTER.decrementAndGet(); + } + + return Collections.emptySet(); + } + + try { + boolean interrupted = false; + while (workerThread.isAlive()) { + workerThread.interrupt(); + try { + workerThread.join(100); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } finally { + INSTANCE_COUNTER.decrementAndGet(); + } + return worker.unprocessedTimeouts(); + } + + @Override + public boolean isStop() { + return WORKER_STATE_SHUTDOWN == WORKER_STATE_UPDATER.get(this); + } + + @Override + public Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) { + if (task == null) { + throw new NullPointerException("task"); + } + if (unit == null) { + throw new NullPointerException("unit"); + } + + long pendingTimeoutsCount = pendingTimeouts.incrementAndGet(); + + if (maxPendingTimeouts > 0 && pendingTimeoutsCount > maxPendingTimeouts) { + pendingTimeouts.decrementAndGet(); + throw new RejectedExecutionException("Number of pending timeouts (" + + pendingTimeoutsCount + ") is greater than or equal to maximum allowed pending " + + "timeouts (" + maxPendingTimeouts + ")"); + } + + start(); + + // Add the timeout to the timeout queue which will be processed on the next tick. + // During processing all the queued HashedWheelTimeouts will be added to the correct HashedWheelBucket. + long deadline = System.nanoTime() + unit.toNanos(delay) - startTime; + + // Guard against overflow. + if (delay > 0 && deadline < 0) { + deadline = Long.MAX_VALUE; + } + HashedWheelTimeout timeout = new HashedWheelTimeout(this, task, deadline); + timeouts.add(timeout); + return timeout; + } + + /** + * Returns the number of pending timeouts of this {@link Timer}. + */ + public long pendingTimeouts() { + return pendingTimeouts.get(); + } + + private static void reportTooManyInstances() { + logger.error("You are creating too many HashedWheelTimer instances. " + + "HashedWheelTimer is a shared resource that must be reused across the JVM," + + "so that only a few instances are created."); + } + + private final class Worker implements Runnable { + private final Set unprocessedTimeouts = new HashSet(); + + private long tick; + + @Override + public void run() { + // Initialize the startTime. + startTime = System.nanoTime(); + if (startTime == 0) { + // We use 0 as an indicator for the uninitialized value here, so make sure it's not 0 when initialized. + startTime = 1; + } + + // Notify the other threads waiting for the initialization at start(). + startTimeInitialized.countDown(); + + do { + final long deadline = waitForNextTick(); + if (deadline > 0) { + int idx = (int) (tick & mask); + processCancelledTasks(); + HashedWheelBucket bucket = + wheel[idx]; + transferTimeoutsToBuckets(); + bucket.expireTimeouts(deadline); + tick++; + } + } while (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_STARTED); + + // Fill the unprocessedTimeouts so we can return them from stop() method. + for (HashedWheelBucket bucket : wheel) { + bucket.clearTimeouts(unprocessedTimeouts); + } + for (; ; ) { + HashedWheelTimeout timeout = timeouts.poll(); + if (timeout == null) { + break; + } + if (!timeout.isCancelled()) { + unprocessedTimeouts.add(timeout); + } + } + processCancelledTasks(); + } + + private void transferTimeoutsToBuckets() { + // transfer only max. 100000 timeouts per tick to prevent a thread to stale the workerThread when it just + // adds new timeouts in a loop. + for (int i = 0; i < 100000; i++) { + HashedWheelTimeout timeout = timeouts.poll(); + if (timeout == null) { + // all processed + break; + } + if (timeout.state() == HashedWheelTimeout.ST_CANCELLED) { + // Was cancelled in the meantime. + continue; + } + + long calculated = timeout.deadline / tickDuration; + timeout.remainingRounds = (calculated - tick) / wheel.length; + + // Ensure we don't schedule for past. + final long ticks = Math.max(calculated, tick); + int stopIndex = (int) (ticks & mask); + + HashedWheelBucket bucket = wheel[stopIndex]; + bucket.addTimeout(timeout); + } + } + + private void processCancelledTasks() { + for (; ; ) { + HashedWheelTimeout timeout = cancelledTimeouts.poll(); + if (timeout == null) { + // all processed + break; + } + try { + timeout.remove(); + } catch (Throwable t) { + if (logger.isWarnEnabled()) { + logger.warn("An exception was thrown while process a cancellation task", t); + } + } + } + } + + /** + * calculate goal nanoTime from startTime and current tick number, + * then wait until that goal has been reached. + * + * @return Long.MIN_VALUE if received a shutdown request, + * current time otherwise (with Long.MIN_VALUE changed by +1) + */ + private long waitForNextTick() { + long deadline = tickDuration * (tick + 1); + + for (; ; ) { + final long currentTime = System.nanoTime() - startTime; + long sleepTimeMs = (deadline - currentTime + 999999) / 1000000; + + if (sleepTimeMs <= 0) { + if (currentTime == Long.MIN_VALUE) { + return -Long.MAX_VALUE; + } else { + return currentTime; + } + } + if (isWindows()) { + sleepTimeMs = sleepTimeMs / 10 * 10; + } + + try { + Thread.sleep(sleepTimeMs); + } catch (InterruptedException ignored) { + if (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_SHUTDOWN) { + return Long.MIN_VALUE; + } + } + } + } + + Set unprocessedTimeouts() { + return Collections.unmodifiableSet(unprocessedTimeouts); + } + } + + private static final class HashedWheelTimeout implements Timeout { + + private static final int ST_INIT = 0; + private static final int ST_CANCELLED = 1; + private static final int ST_EXPIRED = 2; + private static final AtomicIntegerFieldUpdater STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimeout.class, "state"); + + private final HashedWheelTimer timer; + private final TimerTask task; + private final long deadline; + + @SuppressWarnings({"unused", "FieldMayBeFinal", "RedundantFieldInitialization"}) + private volatile int state = ST_INIT; + + /** + * RemainingRounds will be calculated and set by Worker.transferTimeoutsToBuckets() before the + * HashedWheelTimeout will be added to the correct HashedWheelBucket. + */ + long remainingRounds; + + /** + * This will be used to chain timeouts in HashedWheelTimerBucket via a double-linked-list. + * As only the workerThread will act on it there is no need for synchronization / volatile. + */ + HashedWheelTimeout next; + HashedWheelTimeout prev; + + /** + * The bucket to which the timeout was added + */ + HashedWheelBucket bucket; + + HashedWheelTimeout(HashedWheelTimer timer, TimerTask task, long deadline) { + this.timer = timer; + this.task = task; + this.deadline = deadline; + } + + @Override + public Timer timer() { + return timer; + } + + @Override + public TimerTask task() { + return task; + } + + @Override + public boolean cancel() { + // only update the state it will be removed from HashedWheelBucket on next tick. + if (!compareAndSetState(ST_INIT, ST_CANCELLED)) { + return false; + } + // If a task should be canceled we put this to another queue which will be processed on each tick. + // So this means that we will have a GC latency of max. 1 tick duration which is good enough. This way + // we can make again use of our MpscLinkedQueue and so minimize the locking / overhead as much as possible. + timer.cancelledTimeouts.add(this); + return true; + } + + void remove() { + HashedWheelBucket bucket = this.bucket; + if (bucket != null) { + bucket.remove(this); + } else { + timer.pendingTimeouts.decrementAndGet(); + } + } + + public boolean compareAndSetState(int expected, int state) { + return STATE_UPDATER.compareAndSet(this, expected, state); + } + + public int state() { + return state; + } + + @Override + public boolean isCancelled() { + return state() == ST_CANCELLED; + } + + @Override + public boolean isExpired() { + return state() == ST_EXPIRED; + } + + public void expire() { + if (!compareAndSetState(ST_INIT, ST_EXPIRED)) { + return; + } + + try { + task.run(this); + } catch (Throwable t) { + if (logger.isWarnEnabled()) { + logger.warn("An exception was thrown by " + TimerTask.class.getSimpleName() + '.', t); + } + } + } + + @Override + public String toString() { + final long currentTime = System.nanoTime(); + long remaining = deadline - currentTime + timer.startTime; + + StringBuilder buf = new StringBuilder(192) + .append("HashedWheelTimer") + .append('(') + .append("deadline: "); + if (remaining > 0) { + buf.append(remaining) + .append(" ns later"); + } else if (remaining < 0) { + buf.append(-remaining) + .append(" ns ago"); + } else { + buf.append("now"); + } + + if (isCancelled()) { + buf.append(", cancelled"); + } + + return buf.append(", task: ") + .append(task()) + .append(')') + .toString(); + } + } + + /** + * Bucket that stores HashedWheelTimeouts. These are stored in a linked-list like datastructure to allow easy + * removal of HashedWheelTimeouts in the middle. Also the HashedWheelTimeout act as nodes themself and so no + * extra object creation is needed. + */ + private static final class HashedWheelBucket { + + /** + * Used for the linked-list datastructure + */ + private HashedWheelTimeout head; + private HashedWheelTimeout tail; + + /** + * Add {@link HashedWheelTimeout} to this bucket. + */ + void addTimeout(HashedWheelTimeout timeout) { + assert timeout.bucket == null; + timeout.bucket = this; + if (head == null) { + head = tail = timeout; + } else { + tail.next = timeout; + timeout.prev = tail; + tail = timeout; + } + } + + /** + * Expire all {@link HashedWheelTimeout}s for the given {@code deadline}. + */ + void expireTimeouts(long deadline) { + HashedWheelTimeout timeout = head; + + // process all timeouts + while (timeout != null) { + HashedWheelTimeout next = timeout.next; + if (timeout.remainingRounds <= 0) { + next = remove(timeout); + if (timeout.deadline <= deadline) { + timeout.expire(); + } else { + // The timeout was placed into a wrong slot. This should never happen. + throw new IllegalStateException(String.format( + "timeout.deadline (%d) > deadline (%d)", timeout.deadline, deadline)); + } + } else if (timeout.isCancelled()) { + next = remove(timeout); + } else { + timeout.remainingRounds--; + } + timeout = next; + } + } + + public HashedWheelTimeout remove(HashedWheelTimeout timeout) { + HashedWheelTimeout next = timeout.next; + // remove timeout that was either processed or cancelled by updating the linked-list + if (timeout.prev != null) { + timeout.prev.next = next; + } + if (timeout.next != null) { + timeout.next.prev = timeout.prev; + } + + if (timeout == head) { + // if timeout is also the tail we need to adjust the entry too + if (timeout == tail) { + tail = null; + head = null; + } else { + head = next; + } + } else if (timeout == tail) { + // if the timeout is the tail modify the tail to be the prev node. + tail = timeout.prev; + } + // null out prev, next and bucket to allow for GC. + timeout.prev = null; + timeout.next = null; + timeout.bucket = null; + timeout.timer.pendingTimeouts.decrementAndGet(); + return next; + } + + /** + * Clear this bucket and return all not expired / cancelled {@link Timeout}s. + */ + void clearTimeouts(Set set) { + for (; ; ) { + HashedWheelTimeout timeout = pollTimeout(); + if (timeout == null) { + return; + } + if (timeout.isExpired() || timeout.isCancelled()) { + continue; + } + set.add(timeout); + } + } + + private HashedWheelTimeout pollTimeout() { + HashedWheelTimeout head = this.head; + if (head == null) { + return null; + } + HashedWheelTimeout next = head.next; + if (next == null) { + tail = this.head = null; + } else { + this.head = next; + next.prev = null; + } + + // null out prev and next to allow for GC. + head.next = null; + head.prev = null; + head.bucket = null; + return head; + } + } + + private static final boolean IS_OS_WINDOWS = System.getProperty("os.name", "").toLowerCase(Locale.US).contains("win"); + + private boolean isWindows() { + return IS_OS_WINDOWS; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timeout.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timeout.java new file mode 100644 index 0000000..f82a6a3 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timeout.java @@ -0,0 +1,57 @@ +/* + * Copyright 2012 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.usthe.collector.dispatch.timer; + +/** + * A handle associated with a {@link TimerTask} that is returned by a + * {@link Timer}. + * @author from netty | dubbo + */ +@SuppressWarnings("PMD") +public interface Timeout { + + /** + * Returns the {@link Timer} that created this handle. + */ + Timer timer(); + + /** + * Returns the {@link TimerTask} which is associated with this handle. + */ + TimerTask task(); + + /** + * Returns {@code true} if and only if the {@link TimerTask} associated + * with this handle has been expired. + */ + boolean isExpired(); + + /** + * Returns {@code true} if and only if the {@link TimerTask} associated + * with this handle has been cancelled. + */ + boolean isCancelled(); + + /** + * Attempts to cancel the {@link TimerTask} associated with this handle. + * If the task has been executed or cancelled already, it will return with + * no side effect. + * + * @return True if the cancellation completed successfully, otherwise false + */ + boolean cancel(); +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timer.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timer.java new file mode 100644 index 0000000..e962752 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/Timer.java @@ -0,0 +1,58 @@ +/* + * Copyright 2012 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.usthe.collector.dispatch.timer; + +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Schedules {@link TimerTask}s for one-time future execution in a background + * thread. + * @author from netty | dubbo + */ +public interface Timer { + + /** + * Schedules the specified {@link TimerTask} for one-time execution after + * the specified delay. + * + * @param task the {@link TimerTask + * @param delay the delay + * @param unit the unit of time + * @return a handle which is associated with the specified task + * @throws IllegalStateException if this timer has been {@linkplain #stop() stopped} already + * @throws RejectedExecutionException if the pending timeouts are too many and creating new timeout + * can cause instability in the system. + */ + Timeout newTimeout(TimerTask task, long delay, TimeUnit unit); + + /** + * Releases all resources acquired by this {@link Timer} and cancels all + * tasks which were scheduled but not executed yet. + * + * @return the handles associated with the tasks which were canceled by + * this method + */ + Set stop(); + + /** + * the timer is stop + * + * @return true for stop + */ + boolean isStop(); +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatch.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatch.java new file mode 100644 index 0000000..9e9bfd9 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatch.java @@ -0,0 +1,35 @@ +package com.usthe.collector.dispatch.timer; + + +import com.usthe.common.entity.job.Job; + +import java.util.concurrent.TimeUnit; + +/** + * 时间轮调度接口 + * @author tomsun28 + * @date 2021/10/17 22:14 + */ +public interface TimerDispatch { + + /** + * 增加新的job + * @param addJob job + */ + void addJob(Job addJob); + + /** + * 调度循环周期性job + * @param timerTask timerTask + * @param interval 开始调度的间隔时间 + * @param timeUnit 时间单位 + */ + void cyclicJob(WheelTimerJob timerTask, long interval, TimeUnit timeUnit); + + /** + * 删除存在的job + * @param jobId jobId + * @param isCyclic 是否是周期性任务,true是, false为临时性任务 + */ + void deleteJob(long jobId, boolean isCyclic); +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatcher.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatcher.java new file mode 100644 index 0000000..97912f2 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerDispatcher.java @@ -0,0 +1,75 @@ +package com.usthe.collector.dispatch.timer; + +import com.usthe.common.entity.job.Job; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * @author tomsun28 + * @date 2021/10/17 23:06 + */ +@Component +public class TimerDispatcher implements TimerDispatch { + + /** + * 时间轮调度 + */ + private Timer wheelTimer; + /** + * 已存在的周期性调度任务 + */ + private Map currentCyclicTaskMap; + /** + * 已存在的临时性调度任务 + */ + private Map currentTempTaskMap; + + public TimerDispatcher() { + this.wheelTimer = new HashedWheelTimer(r -> { + Thread ret = new Thread(r, "wheelTimer"); + ret.setDaemon(true); + return ret; + }, 10, TimeUnit.SECONDS, 512); + this.currentCyclicTaskMap = new ConcurrentHashMap<>(1024); + this.currentTempTaskMap = new ConcurrentHashMap<>(1024); + } + + @Override + public void addJob(Job addJob) { + WheelTimerJob timerJob = new WheelTimerJob(addJob); + Timeout timeout = wheelTimer.newTimeout(timerJob, addJob.getInterval(), TimeUnit.SECONDS); + if (addJob.isCyclic()) { + currentCyclicTaskMap.put(addJob.getId(), timeout); + } else { + currentTempTaskMap.put(addJob.getId(), timeout); + } + } + + @Override + public void cyclicJob(WheelTimerJob timerTask, long interval, TimeUnit timeUnit) { + Long jobId = timerTask.getJob().getId(); + // 判断此周期性job是否已经被取消 + if (currentCyclicTaskMap.containsKey(jobId)) { + Timeout timeout = wheelTimer.newTimeout(timerTask, interval, TimeUnit.SECONDS); + currentCyclicTaskMap.put(timerTask.getJob().getId(), timeout); + } + } + + @Override + public void deleteJob(long jobId, boolean isCyclic) { + if (isCyclic) { + Timeout timeout = currentCyclicTaskMap.remove(jobId); + if (timeout != null) { + timeout.cancel(); + } + } else { + Timeout timeout = currentTempTaskMap.remove(jobId); + if (timeout != null) { + timeout.cancel(); + } + } + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerTask.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerTask.java new file mode 100644 index 0000000..bec8423 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/TimerTask.java @@ -0,0 +1,36 @@ +/* + * Copyright 2012 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.usthe.collector.dispatch.timer; + +import java.util.concurrent.TimeUnit; + +/** + * A task which is executed after the delay specified with + * {@link Timer#newTimeout(TimerTask, long, TimeUnit)} (TimerTask, long, TimeUnit)}. + * @author from netty | dubbo + */ +public interface TimerTask { + + /** + * Executed after the delay specified with + * {@link Timer#newTimeout(TimerTask, long, TimeUnit)}. + * + * @param timeout a handle which is associated with this task + * @throws Exception when error happen + */ + void run(Timeout timeout) throws Exception; +} diff --git a/collector/server/src/main/java/com/usthe/collector/dispatch/timer/WheelTimerJob.java b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/WheelTimerJob.java new file mode 100644 index 0000000..972a968 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/dispatch/timer/WheelTimerJob.java @@ -0,0 +1,30 @@ +package com.usthe.collector.dispatch.timer; + +import com.usthe.collector.dispatch.MetricsTaskDispatch; +import com.usthe.collector.util.SpringContextHolder; +import com.usthe.common.entity.job.Job; + +/** + * TimerTask实现 + * @author tomsun28 + * @date 2021/11/1 17:18 + */ +public class WheelTimerJob implements TimerTask { + + private Job job; + private MetricsTaskDispatch metricsTaskDispatch; + + public WheelTimerJob(Job job) { + this.job = job; + this.metricsTaskDispatch = SpringContextHolder.getBean(MetricsTaskDispatch.class); + } + + @Override + public void run(Timeout timeout) throws Exception { + metricsTaskDispatch.dispatchMetricsTask(this); + } + + public Job getJob() { + return job; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/plugin/SameClass.java b/collector/server/src/main/java/com/usthe/collector/plugin/SameClass.java new file mode 100644 index 0000000..69fe72d --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/plugin/SameClass.java @@ -0,0 +1,12 @@ +package com.usthe.collector.plugin; + +/** + * @author tomsun28 + * @date 2021/10/8 15:12 + */ +public class SameClass { + + public static String hello() { + return "hello collector"; + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/plugin/TestPlugin.java b/collector/server/src/main/java/com/usthe/collector/plugin/TestPlugin.java new file mode 100644 index 0000000..06efe94 --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/plugin/TestPlugin.java @@ -0,0 +1,20 @@ +package com.usthe.collector.plugin; + +import com.usthe.plugin.sample.ExportDemo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** + * @author tomsun28 + * @date 2021/10/8 15:31 + */ +@Component +@Slf4j +public class TestPlugin implements CommandLineRunner { + @Override + public void run(String... args) throws Exception { + log.info(SameClass.hello()); + log.info(new ExportDemo().hello()); + } +} diff --git a/collector/server/src/main/java/com/usthe/collector/util/SpringContextHolder.java b/collector/server/src/main/java/com/usthe/collector/util/SpringContextHolder.java new file mode 100644 index 0000000..ef3a85f --- /dev/null +++ b/collector/server/src/main/java/com/usthe/collector/util/SpringContextHolder.java @@ -0,0 +1,48 @@ +package com.usthe.collector.util; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +/** + * Spring的ApplicationContext的持有者,可以用静态方法的方式获取spring容器中的bean + * @author tomsun28 + * @date 21:07 2018/4/18 + */ +@Component +public class SpringContextHolder implements ApplicationContextAware { + + private static ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + set(applicationContext); + } + + private static void set(ApplicationContext applicationContext) { + SpringContextHolder.applicationContext = applicationContext; + } + + public static ApplicationContext getApplicationContext() { + assertApplicationContext(); + return applicationContext; + } + + @SuppressWarnings("unchecked") + public static T getBean(String beanName) { + assertApplicationContext(); + return (T) applicationContext.getBean(beanName); + } + + public static T getBean(Class tClass) { + assertApplicationContext(); + return (T) applicationContext.getBean(tClass); + } + + private static void assertApplicationContext() { + if (null == SpringContextHolder.applicationContext) { + throw new RuntimeException("applicationContext为空,请检查是否注入springContextHolder"); + } + } +} diff --git a/collector/server/src/main/resources/application.yml b/collector/server/src/main/resources/application.yml new file mode 100644 index 0000000..047a69a --- /dev/null +++ b/collector/server/src/main/resources/application.yml @@ -0,0 +1,16 @@ +server: + port: 8080 +spring: + application: + name: ${HOSTNAME:@collecor@}${PID} + profiles: + active: dev +collector: + dispatch: + entrance: + etcd: + endpoints: http://139.198.109.64:2379 + export: + kafka: + servers: localhost:9092 + enabled: true diff --git a/collector/server/src/main/resources/banner.txt b/collector/server/src/main/resources/banner.txt new file mode 100644 index 0000000..8e22262 --- /dev/null +++ b/collector/server/src/main/resources/banner.txt @@ -0,0 +1,6 @@ + ██████╗ ██████╗ ██╗ ██╗ ███████╗ ██████╗████████╗ ██████╗ ██████╗ +██╔════╝██╔═══██╗██║ ██║ ██╔════╝██╔════╝╚══██╔══╝██╔═══██╗██╔══██╗ +██║ ██║ ██║██║ ██║ █████╗ ██║ ██║ ██║ ██║██████╔╝ +██║ ██║ ██║██║ ██║ ██╔══╝ ██║ ██║ ██║ ██║██╔══██╗ Profile: ${spring.profiles.active} +╚██████╗╚██████╔╝███████╗███████╗███████╗╚██████╗ ██║ ╚██████╔╝██║ ██║ Name: ${spring.application.name} Port: ${server.port} Pid: ${pid} + ╚═════╝ ╚═════╝ ╚══════╝╚══════╝╚══════╝ ╚═════╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ diff --git a/collector/server/src/main/resources/logback-spring.xml b/collector/server/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..b68b93d --- /dev/null +++ b/collector/server/src/main/resources/logback-spring.xml @@ -0,0 +1,79 @@ + + + + + + + + + 1-%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger - %msg%n + UTF-8 + + + + + + + + logs/${application_name}-%d{yyyy-MM-dd}.%i.log + + + 200MB + + + + true + + + ===%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n + utf-8 + + + + + + logs/${application_name}-%d{yyyy-MM-dd}-error.%i.log + + 200MB + + + + true + + + ===%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n + utf-8 + + + + ERROR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/common/pom.xml b/common/pom.xml new file mode 100644 index 0000000..31f597f --- /dev/null +++ b/common/pom.xml @@ -0,0 +1,35 @@ + + + + monitor + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + common + + + + + + io.etcd + jetcd-core + 0.5.10 + provided + + + + com.google.guava + guava + 31.0.1-jre + + + com.google.code.gson + gson + 2.8.8 + + + \ No newline at end of file diff --git a/common/src/main/java/com/usthe/common/entity/job/Configmap.java b/common/src/main/java/com/usthe/common/entity/job/Configmap.java new file mode 100644 index 0000000..e21bc5f --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/Configmap.java @@ -0,0 +1,35 @@ +package com.usthe.common.entity.job; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 监控配置参数属性及值 + * 过程中需要将协议配置参数里面的标识符为^_^key^_^的内容替换为配置参数里的真实值 + * @author tomsun28 + * @date 2021/10/29 22:04 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class Configmap { + + /** + * 参数key,将协议配置参数里面的标识符为^^_key_^^的内容替换为配置参数里的真实值 + */ + private String key; + + /** + * 参数value + */ + private Object value; + + /** + * number,string,secret + * 数字,非加密字符串,加密字符串 + */ + private String type; +} diff --git a/common/src/main/java/com/usthe/common/entity/job/Job.java b/common/src/main/java/com/usthe/common/entity/job/Job.java new file mode 100644 index 0000000..f87413b --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/Job.java @@ -0,0 +1,162 @@ +package com.usthe.common.entity.job; + + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 采集任务详情 + * @author tomsun28 + * @date 2021/10/17 21:19 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Slf4j +public class Job { + + private static final String AVAILABILITY = "availability"; + + /** + * 任务ID + */ + private long id; + /** + * 监控ID 应用ID + */ + private long appId; + /** + * 监控的类型 eg: linux | mysql | jvm + */ + private String app; + /** + * 任务派发开始时间戳 + */ + private long timestamp; + /** + * 任务采集时间间隔(单位秒) eg: 30,60,600 + */ + private long interval; + /** + * 是否是循环周期性任务 true为是,false为否 + */ + private boolean isCyclic; + /** + * 指标组配置 eg: cpu memory + */ + private List metrics; + /** + * 监控配置参数属性及值 eg: username password timeout host + */ + private List configmap; + + /** + * collector使用 - 任务版本,此字段不存储于etcd + */ + private transient long version; + /** + * collector使用 - 指标组任务执行优先级视图 + * 0 - availability + * 1 - cpu | memory + * 2 - health + * 3 - otherMetrics + * .... + * 126 - otherMetrics + * 127 - lastPriorMetrics + */ + private transient List> priorMetrics; + + /** + * collector使用 - 构造初始化标志 + */ + private transient boolean isConstruct = false; + + /** + * collector使用 - 构造初始化指标组 + */ + public synchronized void constructMetrics() { + if (isConstruct) { + return; + } + Map> map = metrics.stream() + .peek(metric -> { + // 判断是否配置aliasFields 没有则配置默认 + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields()); + } + // 设置默认的指标组执行优先级 + if (metric.getPriority() == null) { + if (AVAILABILITY.equals(metric.getName())) { + metric.setPriority((byte)0); + } else { + metric.setPriority(Byte.MAX_VALUE); + } + } + }) + .collect(Collectors.groupingBy(Metrics::getPriority)); + // 构造指标组任务执行顺序链表 + priorMetrics = new LinkedList<>(); + map.values().forEach(metric -> { + Set metricsSet = new HashSet<>(metric); + priorMetrics.add(metricsSet); + }); + priorMetrics.sort(Comparator.comparing(e -> { + Optional metric = e.stream().findAny(); + if (metric.isPresent()) { + return metric.get().getPriority(); + } else { + return Byte.MAX_VALUE; + } + })); + } + + /** + * collector使用 - 获取下一组优先级的指标组任务 + * @param metrics 当前指标组 + * @param first 是否是第一次获取 + * @return 指标组任务 + * 返回null表示：job已完成,所有指标组采集结束 + * 返回empty的集合表示：当前级别下还有指标组采集任务未结束,无法进行下一级别的指标组任务采集 + * 返回有数据集合表示：获取到下一组优先级的指标组任务 + */ + public synchronized Set getNextCollectMetrics(Metrics metrics, boolean first) { + if (!isConstruct || priorMetrics == null || priorMetrics.isEmpty()) { + return null; + } + Set metricsSet = priorMetrics.get(0); + if (first) { + log.error("metrics must has one [availability] metrics at least."); + return metricsSet; + } + if (metrics == null) { + log.error("metrics can not null when not first get"); + } + if (metrics != null && !metricsSet.remove(metrics)) { + log.error("Job {} appId {} app {} metrics {} remove empty error in priorMetrics.", + id, appId, app, metrics.getName()); + } + if (metricsSet.isEmpty()) { + if (priorMetrics.size() == 1) { + return null; + } + priorMetrics.remove(0); + return priorMetrics.get(0); + } else { + return Collections.emptySet(); + } + } +} diff --git a/common/src/main/java/com/usthe/common/entity/job/Metrics.java b/common/src/main/java/com/usthe/common/entity/job/Metrics.java new file mode 100644 index 0000000..dc1fd4a --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/Metrics.java @@ -0,0 +1,70 @@ +package com.usthe.common.entity.job; + +import com.usthe.common.entity.job.protocol.HttpProtocol; +import com.usthe.common.entity.job.protocol.IcmpProtocol; +import com.usthe.common.entity.job.protocol.JdbcProtocol; +import com.usthe.common.entity.job.protocol.TcpUdpProtocol; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * 监控采集的指标集合详情 eg: cpu | memory | health + * @author tomsun28 + * @date 2021/10/17 21:24 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class Metrics { + + /** + * 公共属性-名称 eg: cpu | memory | health + */ + private String name; + /** + * 公共属性-采集监控协议 eg: sql, ssh, http, telnet, wmi, snmp, sdk + */ + private String protocol; + /** + * 范围(0-127)指标组调度优先级,数值越小优先级越高 + * 优先级高的调度采集完成后才会调度下一优先级的指标组采集任务 + * 可用性指标组(availability)默认优先级为0,其它普通指标组范围为1-127,即需要等availability采集成功后才会调度后面的指标组任务 + */ + private Byte priority; + /** + * 公共属性-采集监控的最终结果属性集合 eg: speed | times | size + */ + private List fields; + /** + * 公共属性-采集监控的前置查询属性集合 eg: size1 | size2 | speedSize + */ + private List aliasFields; + /** + * 公共属性-表达式计算，将前置查询属性(preFields)与最终属性(fields)映射,计算出最终属性(fields)值 + * eg: size = size1 + size2, speed = speedSize + * https://www.yuque.com/boyan-avfmj/aviatorscript/ban32m + */ + private List calculates; + + /** + * 使用http协议的监控配置信息 + */ + private HttpProtocol http; + /** + * 使用icmp协议进行ping的监控配置信息 + */ + private IcmpProtocol icmp; + /** + * 使用socket实现的tcp或ucp进行服务端口探测配置信息 + */ + private TcpUdpProtocol tcpUdp; + /** + * 使用公共的jdbc规范实现的数据库配置信息 + */ + private JdbcProtocol jdbc; +} diff --git a/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java b/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java new file mode 100644 index 0000000..95cd54c --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/protocol/HttpProtocol.java @@ -0,0 +1,93 @@ +package com.usthe.common.entity.job.protocol; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +/** + * http 协议配置 + * @author tomsun28 + * @date 2021/10/31 12:41 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class HttpProtocol { + /** + * 对端主机ip或域名 + */ + private String host; + /** + * http/https 请求访问的url链接 + */ + private String url; + /** + * http是否使用链路加密ssl/tls,即是http还是https + */ + private boolean ssl = false; + /** + * http请求方法: get, post, put, delete, patch + */ + private String method; + /** + * http请求携带头 eg: Content-Type = application/json + */ + private Map headers; + /** + * http请求携带查询参数 eg: localhost:80/api?paramKey=value + */ + private Map params; + /** + * 认证信息 + */ + private Authorization authorization; + /** + * 响应数据解析方式 + * default - 自有的数据解析规则 + * json_path 自定义jsonPath脚本 https://www.jsonpath.cn/ + * xml_path 自定义xmlPath脚本 + * prometheus Prometheus数据规则 + */ + private String parseType; + /** + * 数据解析脚本 当解析方式为 jsonPath or xmlPath时存在 + */ + private String parseScript; + + /** + * 认证信息 + */ + @Data + @AllArgsConstructor + @NoArgsConstructor + public class Authorization { + /** + * 认证类型：Bearer Token, Basic Auth, Digest Auth + */ + private String type; + /** + * Bearer Token's token + */ + private String bearerTokenToken; + /** + * Basic Auth's username + */ + private String basicAuthUsername; + /** + * Basic Auth's password + */ + private String basicAuthPassword; + /** + * Digest Auth's username + */ + private String digestAuthUsername; + /** + * Digest Auth's password + */ + private String digestAuthPassword; + } +} diff --git a/common/src/main/java/com/usthe/common/entity/job/protocol/IcmpProtocol.java b/common/src/main/java/com/usthe/common/entity/job/protocol/IcmpProtocol.java new file mode 100644 index 0000000..f9ba6d4 --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/protocol/IcmpProtocol.java @@ -0,0 +1,23 @@ +package com.usthe.common.entity.job.protocol; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * icmp(即ping)协议配置 + * @author tomsun28 + * @date 2021/10/31 16:41 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class IcmpProtocol { + /** + * 对端主机ip或域名 + */ + private String host; + +} diff --git a/common/src/main/java/com/usthe/common/entity/job/protocol/JdbcProtocol.java b/common/src/main/java/com/usthe/common/entity/job/protocol/JdbcProtocol.java new file mode 100644 index 0000000..4c61915 --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/protocol/JdbcProtocol.java @@ -0,0 +1,38 @@ +package com.usthe.common.entity.job.protocol; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 公共的jdbc规范实现的数据库配置信息 + * @author tomsun28 + * @date 2021/10/31 17:33 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class JdbcProtocol { + /** + * 对端主机ip或域名 + */ + private String host; + /** + * 端口号 + */ + private Integer port; + /** + * 数据库用户名(可选) + */ + private String username; + /** + * 数据库密码(可选) + */ + private String password; + /** + * 数据库链接url eg: jdbc:mysql://localhost:3306 + */ + private String url; +} diff --git a/common/src/main/java/com/usthe/common/entity/job/protocol/TcpUdpProtocol.java b/common/src/main/java/com/usthe/common/entity/job/protocol/TcpUdpProtocol.java new file mode 100644 index 0000000..3f2afd0 --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/job/protocol/TcpUdpProtocol.java @@ -0,0 +1,30 @@ +package com.usthe.common.entity.job.protocol; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 使用socket实现的tcp或ucp进行服务端口可用性探测 + * @author tomsun28 + * @date 2021/10/31 17:27 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class TcpUdpProtocol { + /** + * 具体协议类型 tcp, udp + */ + private String protocol; + /** + * 对端主机ip或域名 + */ + private String host; + /** + * 端口号 + */ + private Integer port; +} diff --git a/common/src/main/java/com/usthe/common/entity/message/CollectRep.java b/common/src/main/java/com/usthe/common/entity/message/CollectRep.java new file mode 100644 index 0000000..a990c4d --- /dev/null +++ b/common/src/main/java/com/usthe/common/entity/message/CollectRep.java @@ -0,0 +1,3225 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: collect_rep.proto + +package com.usthe.common.entity.message; + +@SuppressWarnings("PMD") +public final class CollectRep { + private CollectRep() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions( + (com.google.protobuf.ExtensionRegistryLite) registry); + } + /** + * Protobuf enum {@code com.usthe.common.entity.message.Code} + */ + public enum Code + implements com.google.protobuf.ProtocolMessageEnum { + /** + * 

+     * 采集成功
+     *
+ * + * SUCCESS = 0; + */ + SUCCESS(0), + /** + * 
+     * 采集器不可用
+     *
+ * + * UN_AVAILABLE = 1; + */ + UN_AVAILABLE(1), + /** + * 
+     * 对端不可达(网络层icmp)
+     *
+ * + * UN_REACHABLE = 2; + */ + UN_REACHABLE(2), + /** + * 
+     * 对端连接失败(传输层tcp,udp)
+     *
+ * + * UN_CONNECTABLE = 3; + */ + UN_CONNECTABLE(3), + /** + * 
+     * 数据采集失败(应用层http,ssh,snmp)
+     *
+ * + * FAIL = 4; + */ + FAIL(4), + /** + * 
+     * 采集超时
+     *
+ * + * TIMEOUT = 5; + */ + TIMEOUT(5), + UNRECOGNIZED(-1), + ; + + /** + * 
+     * 采集成功
+     *
+ * + * SUCCESS = 0; + */ + public static final int SUCCESS_VALUE = 0; + /** + * 
+     * 采集器不可用
+     *
+ * + * UN_AVAILABLE = 1; + */ + public static final int UN_AVAILABLE_VALUE = 1; + /** + * 
+     * 对端不可达(网络层icmp)
+     *
+ * + * UN_REACHABLE = 2; + */ + public static final int UN_REACHABLE_VALUE = 2; + /** + * 
+     * 对端连接失败(传输层tcp,udp)
+     *
+ * + * UN_CONNECTABLE = 3; + */ + public static final int UN_CONNECTABLE_VALUE = 3; + /** + * 
+     * 数据采集失败(应用层http,ssh,snmp)
+     *
+ * + * FAIL = 4; + */ + public static final int FAIL_VALUE = 4; + /** + * 
+     * 采集超时
+     *
+ * + * TIMEOUT = 5; + */ + public static final int TIMEOUT_VALUE = 5; + + + public final int getNumber() { + if (this == UNRECOGNIZED) { + throw new java.lang.IllegalArgumentException( + "Can't get the number of an unknown enum value."); + } + return value; + } + + /** + * @param value The numeric wire value of the corresponding enum entry. + * @return The enum associated with the given numeric wire value. + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static Code valueOf(int value) { + return forNumber(value); + } + + /** + * @param value The numeric wire value of the corresponding enum entry. + * @return The enum associated with the given numeric wire value. + */ + public static Code forNumber(int value) { + switch (value) { + case 0: return SUCCESS; + case 1: return UN_AVAILABLE; + case 2: return UN_REACHABLE; + case 3: return UN_CONNECTABLE; + case 4: return FAIL; + case 5: return TIMEOUT; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static final com.google.protobuf.Internal.EnumLiteMap< + Code> internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public Code findValueByNumber(int number) { + return Code.forNumber(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + if (this == UNRECOGNIZED) { + throw new java.lang.IllegalStateException( + "Can't get the descriptor of an unrecognized enum value."); + } + return getDescriptor().getValues().get(ordinal()); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return com.usthe.common.entity.message.CollectRep.getDescriptor().getEnumTypes().get(0); + } + + private static final Code[] VALUES = values(); + + public static Code valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + if (desc.getIndex() == -1) { + return UNRECOGNIZED; + } + return VALUES[desc.getIndex()]; + } + + private final int value; + + private Code(int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:com.usthe.common.entity.message.Code) + } + + public interface MetricsDataOrBuilder extends + // @@protoc_insertion_point(interface_extends:com.usthe.common.entity.message.MetricsData) + com.google.protobuf.MessageOrBuilder { + + /** + * 
+     * 监控的ID
+     *
+ * + * uint64 id = 1; + * @return The id. + */ + long getId(); + + /** + * 
+     * 监控的类型 eg: linux | mysql | jvm
+     *
+ * + * string app = 2; + * @return The app. + */ + java.lang.String getApp(); + /** + * 
+     * 监控的类型 eg: linux | mysql | jvm
+     *
+ * + * string app = 2; + * @return The bytes for app. + */ + com.google.protobuf.ByteString + getAppBytes(); + + /** + * 
+     * 监控采集的指标集合 eg: cpu | memory | health
+     *
+ * + * string metrics = 3; + * @return The metrics. + */ + java.lang.String getMetrics(); + /** + * 
+     * 监控采集的指标集合 eg: cpu | memory | health
+     *
+ * + * string metrics = 3; + * @return The bytes for metrics. + */ + com.google.protobuf.ByteString + getMetricsBytes(); + + /** + * 
+     * 采集时间
+     *
+ * + * uint64 time = 4; + * @return The time. + */ + long getTime(); + + /** + * 
+     * 采集响应码
+     *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return The enum numeric value on the wire for code. + */ + int getCodeValue(); + /** + * 
+     * 采集响应码
+     *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return The code. + */ + com.usthe.common.entity.message.CollectRep.Code getCode(); + + /** + * 
+     * 采集响应信息
+     *
+ * + * string msg = 6; + * @return The msg. + */ + java.lang.String getMsg(); + /** + * 
+     * 采集响应信息
+     *
+ * + * string msg = 6; + * @return The bytes for msg. + */ + com.google.protobuf.ByteString + getMsgBytes(); + + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @return A list containing the fields. + */ + java.util.List + getFieldsList(); + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @return The count of fields. + */ + int getFieldsCount(); + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @param index The index of the element to return. + * @return The fields at the given index. + */ + java.lang.String getFields(int index); + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @param index The index of the value to return. + * @return The bytes of the fields at the given index. + */ + com.google.protobuf.ByteString + getFieldsBytes(int index); + + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + java.util.List + getValuesList(); + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + com.usthe.common.entity.message.CollectRep.ValueRow getValues(int index); + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + int getValuesCount(); + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + java.util.List + getValuesOrBuilderList(); + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder getValuesOrBuilder( + int index); + } + /** + * Protobuf type {@code com.usthe.common.entity.message.MetricsData} + */ + public static final class MetricsData extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:com.usthe.common.entity.message.MetricsData) + MetricsDataOrBuilder { + private static final long serialVersionUID = 0L; + // Use MetricsData.newBuilder() to construct. + private MetricsData(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private MetricsData() { + app_ = ""; + metrics_ = ""; + code_ = 0; + msg_ = ""; + fields_ = com.google.protobuf.LazyStringArrayList.EMPTY; + values_ = java.util.Collections.emptyList(); + } + + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance( + UnusedPrivateParameter unused) { + return new MetricsData(); + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private MetricsData( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 8: { + + id_ = input.readUInt64(); + break; + } + case 18: { + java.lang.String s = input.readStringRequireUtf8(); + + app_ = s; + break; + } + case 26: { + java.lang.String s = input.readStringRequireUtf8(); + + metrics_ = s; + break; + } + case 32: { + + time_ = input.readUInt64(); + break; + } + case 40: { + int rawValue = input.readEnum(); + + code_ = rawValue; + break; + } + case 50: { + java.lang.String s = input.readStringRequireUtf8(); + + msg_ = s; + break; + } + case 58: { + java.lang.String s = input.readStringRequireUtf8(); + if (!((mutable_bitField0_ & 0x00000001) != 0)) { + fields_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + fields_.add(s); + break; + } + case 66: { + if (!((mutable_bitField0_ & 0x00000002) != 0)) { + values_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000002; + } + values_.add( + input.readMessage(com.usthe.common.entity.message.CollectRep.ValueRow.parser(), extensionRegistry)); + break; + } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) != 0)) { + fields_ = fields_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) != 0)) { + values_ = java.util.Collections.unmodifiableList(values_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_MetricsData_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_MetricsData_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.usthe.common.entity.message.CollectRep.MetricsData.class, com.usthe.common.entity.message.CollectRep.MetricsData.Builder.class); + } + + public static final int ID_FIELD_NUMBER = 1; + private long id_; + /** + * 
+     * 监控的ID
+     *
+ * + * uint64 id = 1; + * @return The id. + */ + @java.lang.Override + public long getId() { + return id_; + } + + public static final int APP_FIELD_NUMBER = 2; + private volatile java.lang.Object app_; + /** + * 
+     * 监控的类型 eg: linux | mysql | jvm
+     *
+ * + * string app = 2; + * @return The app. + */ + @java.lang.Override + public java.lang.String getApp() { + java.lang.Object ref = app_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + app_ = s; + return s; + } + } + /** + * 
+     * 监控的类型 eg: linux | mysql | jvm
+     *
+ * + * string app = 2; + * @return The bytes for app. + */ + @java.lang.Override + public com.google.protobuf.ByteString + getAppBytes() { + java.lang.Object ref = app_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + app_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int METRICS_FIELD_NUMBER = 3; + private volatile java.lang.Object metrics_; + /** + * 
+     * 监控采集的指标集合 eg: cpu | memory | health
+     *
+ * + * string metrics = 3; + * @return The metrics. + */ + @java.lang.Override + public java.lang.String getMetrics() { + java.lang.Object ref = metrics_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + metrics_ = s; + return s; + } + } + /** + * 
+     * 监控采集的指标集合 eg: cpu | memory | health
+     *
+ * + * string metrics = 3; + * @return The bytes for metrics. + */ + @java.lang.Override + public com.google.protobuf.ByteString + getMetricsBytes() { + java.lang.Object ref = metrics_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + metrics_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TIME_FIELD_NUMBER = 4; + private long time_; + /** + * 
+     * 采集时间
+     *
+ * + * uint64 time = 4; + * @return The time. + */ + @java.lang.Override + public long getTime() { + return time_; + } + + public static final int CODE_FIELD_NUMBER = 5; + private int code_; + /** + * 
+     * 采集响应码
+     *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return The enum numeric value on the wire for code. + */ + @java.lang.Override public int getCodeValue() { + return code_; + } + /** + * 
+     * 采集响应码
+     *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return The code. + */ + @java.lang.Override public com.usthe.common.entity.message.CollectRep.Code getCode() { + @SuppressWarnings("deprecation") + com.usthe.common.entity.message.CollectRep.Code result = com.usthe.common.entity.message.CollectRep.Code.valueOf(code_); + return result == null ? com.usthe.common.entity.message.CollectRep.Code.UNRECOGNIZED : result; + } + + public static final int MSG_FIELD_NUMBER = 6; + private volatile java.lang.Object msg_; + /** + * 
+     * 采集响应信息
+     *
+ * + * string msg = 6; + * @return The msg. + */ + @java.lang.Override + public java.lang.String getMsg() { + java.lang.Object ref = msg_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + msg_ = s; + return s; + } + } + /** + * 
+     * 采集响应信息
+     *
+ * + * string msg = 6; + * @return The bytes for msg. + */ + @java.lang.Override + public com.google.protobuf.ByteString + getMsgBytes() { + java.lang.Object ref = msg_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + msg_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int FIELDS_FIELD_NUMBER = 7; + private com.google.protobuf.LazyStringList fields_; + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @return A list containing the fields. + */ + public com.google.protobuf.ProtocolStringList + getFieldsList() { + return fields_; + } + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @return The count of fields. + */ + public int getFieldsCount() { + return fields_.size(); + } + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @param index The index of the element to return. + * @return The fields at the given index. + */ + public java.lang.String getFields(int index) { + return fields_.get(index); + } + /** + * 
+     * 采集指标名
+     *
+ * + * repeated string fields = 7; + * @param index The index of the value to return. + * @return The bytes of the fields at the given index. + */ + public com.google.protobuf.ByteString + getFieldsBytes(int index) { + return fields_.getByteString(index); + } + + public static final int VALUES_FIELD_NUMBER = 8; + private java.util.List values_; + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + @java.lang.Override + public java.util.List getValuesList() { + return values_; + } + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + @java.lang.Override + public java.util.List + getValuesOrBuilderList() { + return values_; + } + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + @java.lang.Override + public int getValuesCount() { + return values_.size(); + } + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.ValueRow getValues(int index) { + return values_.get(index); + } + /** + * 
+     * 采集指标值集合(fields作为字段名称与ValueRow映射)
+     *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder getValuesOrBuilder( + int index) { + return values_.get(index); + } + + private byte memoizedIsInitialized = -1; + @java.lang.Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + @java.lang.Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (id_ != 0L) { + output.writeUInt64(1, id_); + } + if (!getAppBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, app_); + } + if (!getMetricsBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, metrics_); + } + if (time_ != 0L) { + output.writeUInt64(4, time_); + } + if (code_ != com.usthe.common.entity.message.CollectRep.Code.SUCCESS.getNumber()) { + output.writeEnum(5, code_); + } + if (!getMsgBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 6, msg_); + } + for (int i = 0; i < fields_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 7, fields_.getRaw(i)); + } + for (int i = 0; i < values_.size(); i++) { + output.writeMessage(8, values_.get(i)); + } + unknownFields.writeTo(output); + } + + @java.lang.Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (id_ != 0L) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(1, id_); + } + if (!getAppBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, app_); + } + if (!getMetricsBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, metrics_); + } + if (time_ != 0L) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(4, time_); + } + if (code_ != com.usthe.common.entity.message.CollectRep.Code.SUCCESS.getNumber()) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(5, code_); + } + if (!getMsgBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(6, msg_); + } + { + int dataSize = 0; + for (int i = 0; i < fields_.size(); i++) { + dataSize += computeStringSizeNoTag(fields_.getRaw(i)); + } + size += dataSize; + size += 1 * getFieldsList().size(); + } + for (int i = 0; i < values_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(8, values_.get(i)); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.usthe.common.entity.message.CollectRep.MetricsData)) { + return super.equals(obj); + } + com.usthe.common.entity.message.CollectRep.MetricsData other = (com.usthe.common.entity.message.CollectRep.MetricsData) obj; + + if (getId() + != other.getId()) return false; + if (!getApp() + .equals(other.getApp())) return false; + if (!getMetrics() + .equals(other.getMetrics())) return false; + if (getTime() + != other.getTime()) return false; + if (code_ != other.code_) return false; + if (!getMsg() + .equals(other.getMsg())) return false; + if (!getFieldsList() + .equals(other.getFieldsList())) return false; + if (!getValuesList() + .equals(other.getValuesList())) return false; + if (!unknownFields.equals(other.unknownFields)) return false; + return true; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + ID_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong( + getId()); + hash = (37 * hash) + APP_FIELD_NUMBER; + hash = (53 * hash) + getApp().hashCode(); + hash = (37 * hash) + METRICS_FIELD_NUMBER; + hash = (53 * hash) + getMetrics().hashCode(); + hash = (37 * hash) + TIME_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong( + getTime()); + hash = (37 * hash) + CODE_FIELD_NUMBER; + hash = (53 * hash) + code_; + hash = (37 * hash) + MSG_FIELD_NUMBER; + hash = (53 * hash) + getMsg().hashCode(); + if (getFieldsCount() > 0) { + hash = (37 * hash) + FIELDS_FIELD_NUMBER; + hash = (53 * hash) + getFieldsList().hashCode(); + } + if (getValuesCount() > 0) { + hash = (37 * hash) + VALUES_FIELD_NUMBER; + hash = (53 * hash) + getValuesList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static com.usthe.common.entity.message.CollectRep.MetricsData parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @java.lang.Override + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(com.usthe.common.entity.message.CollectRep.MetricsData prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + @java.lang.Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code com.usthe.common.entity.message.MetricsData} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:com.usthe.common.entity.message.MetricsData) + com.usthe.common.entity.message.CollectRep.MetricsDataOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_MetricsData_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_MetricsData_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.usthe.common.entity.message.CollectRep.MetricsData.class, com.usthe.common.entity.message.CollectRep.MetricsData.Builder.class); + } + + // Construct using com.usthe.common.entity.message.CollectRep.MetricsData.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + getValuesFieldBuilder(); + } + } + @java.lang.Override + public Builder clear() { + super.clear(); + id_ = 0L; + + app_ = ""; + + metrics_ = ""; + + time_ = 0L; + + code_ = 0; + + msg_ = ""; + + fields_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + if (valuesBuilder_ == null) { + values_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + valuesBuilder_.clear(); + } + return this; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_MetricsData_descriptor; + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.MetricsData getDefaultInstanceForType() { + return com.usthe.common.entity.message.CollectRep.MetricsData.getDefaultInstance(); + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.MetricsData build() { + com.usthe.common.entity.message.CollectRep.MetricsData result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.MetricsData buildPartial() { + com.usthe.common.entity.message.CollectRep.MetricsData result = new com.usthe.common.entity.message.CollectRep.MetricsData(this); + int from_bitField0_ = bitField0_; + result.id_ = id_; + result.app_ = app_; + result.metrics_ = metrics_; + result.time_ = time_; + result.code_ = code_; + result.msg_ = msg_; + if (((bitField0_ & 0x00000001) != 0)) { + fields_ = fields_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.fields_ = fields_; + if (valuesBuilder_ == null) { + if (((bitField0_ & 0x00000002) != 0)) { + values_ = java.util.Collections.unmodifiableList(values_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.values_ = values_; + } else { + result.values_ = valuesBuilder_.build(); + } + onBuilt(); + return result; + } + + @java.lang.Override + public Builder clone() { + return super.clone(); + } + @java.lang.Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return super.setField(field, value); + } + @java.lang.Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return super.clearField(field); + } + @java.lang.Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return super.clearOneof(oneof); + } + @java.lang.Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, java.lang.Object value) { + return super.setRepeatedField(field, index, value); + } + @java.lang.Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return super.addRepeatedField(field, value); + } + @java.lang.Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.usthe.common.entity.message.CollectRep.MetricsData) { + return mergeFrom((com.usthe.common.entity.message.CollectRep.MetricsData)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.usthe.common.entity.message.CollectRep.MetricsData other) { + if (other == com.usthe.common.entity.message.CollectRep.MetricsData.getDefaultInstance()) return this; + if (other.getId() != 0L) { + setId(other.getId()); + } + if (!other.getApp().isEmpty()) { + app_ = other.app_; + onChanged(); + } + if (!other.getMetrics().isEmpty()) { + metrics_ = other.metrics_; + onChanged(); + } + if (other.getTime() != 0L) { + setTime(other.getTime()); + } + if (other.code_ != 0) { + setCodeValue(other.getCodeValue()); + } + if (!other.getMsg().isEmpty()) { + msg_ = other.msg_; + onChanged(); + } + if (!other.fields_.isEmpty()) { + if (fields_.isEmpty()) { + fields_ = other.fields_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureFieldsIsMutable(); + fields_.addAll(other.fields_); + } + onChanged(); + } + if (valuesBuilder_ == null) { + if (!other.values_.isEmpty()) { + if (values_.isEmpty()) { + values_ = other.values_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureValuesIsMutable(); + values_.addAll(other.values_); + } + onChanged(); + } + } else { + if (!other.values_.isEmpty()) { + if (valuesBuilder_.isEmpty()) { + valuesBuilder_.dispose(); + valuesBuilder_ = null; + values_ = other.values_; + bitField0_ = (bitField0_ & ~0x00000002); + valuesBuilder_ = + com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? + getValuesFieldBuilder() : null; + } else { + valuesBuilder_.addAllMessages(other.values_); + } + } + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @java.lang.Override + public final boolean isInitialized() { + return true; + } + + @java.lang.Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.usthe.common.entity.message.CollectRep.MetricsData parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.usthe.common.entity.message.CollectRep.MetricsData) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + private long id_ ; + /** + * 
+       * 监控的ID
+       *
+ * + * uint64 id = 1; + * @return The id. + */ + @java.lang.Override + public long getId() { + return id_; + } + /** + * 
+       * 监控的ID
+       *
+ * + * uint64 id = 1; + * @param value The id to set. + * @return This builder for chaining. + */ + public Builder setId(long value) { + + id_ = value; + onChanged(); + return this; + } + /** + * 
+       * 监控的ID
+       *
+ * + * uint64 id = 1; + * @return This builder for chaining. + */ + public Builder clearId() { + + id_ = 0L; + onChanged(); + return this; + } + + private java.lang.Object app_ = ""; + /** + * 
+       * 监控的类型 eg: linux | mysql | jvm
+       *
+ * + * string app = 2; + * @return The app. + */ + public java.lang.String getApp() { + java.lang.Object ref = app_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + app_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * 
+       * 监控的类型 eg: linux | mysql | jvm
+       *
+ * + * string app = 2; + * @return The bytes for app. + */ + public com.google.protobuf.ByteString + getAppBytes() { + java.lang.Object ref = app_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + app_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * 
+       * 监控的类型 eg: linux | mysql | jvm
+       *
+ * + * string app = 2; + * @param value The app to set. + * @return This builder for chaining. + */ + public Builder setApp( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + app_ = value; + onChanged(); + return this; + } + /** + * 
+       * 监控的类型 eg: linux | mysql | jvm
+       *
+ * + * string app = 2; + * @return This builder for chaining. + */ + public Builder clearApp() { + + app_ = getDefaultInstance().getApp(); + onChanged(); + return this; + } + /** + * 
+       * 监控的类型 eg: linux | mysql | jvm
+       *
+ * + * string app = 2; + * @param value The bytes for app to set. + * @return This builder for chaining. + */ + public Builder setAppBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + app_ = value; + onChanged(); + return this; + } + + private java.lang.Object metrics_ = ""; + /** + * 
+       * 监控采集的指标集合 eg: cpu | memory | health
+       *
+ * + * string metrics = 3; + * @return The metrics. + */ + public java.lang.String getMetrics() { + java.lang.Object ref = metrics_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + metrics_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * 
+       * 监控采集的指标集合 eg: cpu | memory | health
+       *
+ * + * string metrics = 3; + * @return The bytes for metrics. + */ + public com.google.protobuf.ByteString + getMetricsBytes() { + java.lang.Object ref = metrics_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + metrics_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * 
+       * 监控采集的指标集合 eg: cpu | memory | health
+       *
+ * + * string metrics = 3; + * @param value The metrics to set. + * @return This builder for chaining. + */ + public Builder setMetrics( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + metrics_ = value; + onChanged(); + return this; + } + /** + * 
+       * 监控采集的指标集合 eg: cpu | memory | health
+       *
+ * + * string metrics = 3; + * @return This builder for chaining. + */ + public Builder clearMetrics() { + + metrics_ = getDefaultInstance().getMetrics(); + onChanged(); + return this; + } + /** + * 
+       * 监控采集的指标集合 eg: cpu | memory | health
+       *
+ * + * string metrics = 3; + * @param value The bytes for metrics to set. + * @return This builder for chaining. + */ + public Builder setMetricsBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + metrics_ = value; + onChanged(); + return this; + } + + private long time_ ; + /** + * 
+       * 采集时间
+       *
+ * + * uint64 time = 4; + * @return The time. + */ + @java.lang.Override + public long getTime() { + return time_; + } + /** + * 
+       * 采集时间
+       *
+ * + * uint64 time = 4; + * @param value The time to set. + * @return This builder for chaining. + */ + public Builder setTime(long value) { + + time_ = value; + onChanged(); + return this; + } + /** + * 
+       * 采集时间
+       *
+ * + * uint64 time = 4; + * @return This builder for chaining. + */ + public Builder clearTime() { + + time_ = 0L; + onChanged(); + return this; + } + + private int code_ = 0; + /** + * 
+       * 采集响应码
+       *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return The enum numeric value on the wire for code. + */ + @java.lang.Override public int getCodeValue() { + return code_; + } + /** + * 
+       * 采集响应码
+       *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @param value The enum numeric value on the wire for code to set. + * @return This builder for chaining. + */ + public Builder setCodeValue(int value) { + + code_ = value; + onChanged(); + return this; + } + /** + * 
+       * 采集响应码
+       *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return The code. + */ + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.Code getCode() { + @SuppressWarnings("deprecation") + com.usthe.common.entity.message.CollectRep.Code result = com.usthe.common.entity.message.CollectRep.Code.valueOf(code_); + return result == null ? com.usthe.common.entity.message.CollectRep.Code.UNRECOGNIZED : result; + } + /** + * 
+       * 采集响应码
+       *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @param value The code to set. + * @return This builder for chaining. + */ + public Builder setCode(com.usthe.common.entity.message.CollectRep.Code value) { + if (value == null) { + throw new NullPointerException(); + } + + code_ = value.getNumber(); + onChanged(); + return this; + } + /** + * 
+       * 采集响应码
+       *
+ * + * .com.usthe.common.entity.message.Code code = 5; + * @return This builder for chaining. + */ + public Builder clearCode() { + + code_ = 0; + onChanged(); + return this; + } + + private java.lang.Object msg_ = ""; + /** + * 
+       * 采集响应信息
+       *
+ * + * string msg = 6; + * @return The msg. + */ + public java.lang.String getMsg() { + java.lang.Object ref = msg_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + msg_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * 
+       * 采集响应信息
+       *
+ * + * string msg = 6; + * @return The bytes for msg. + */ + public com.google.protobuf.ByteString + getMsgBytes() { + java.lang.Object ref = msg_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + msg_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * 
+       * 采集响应信息
+       *
+ * + * string msg = 6; + * @param value The msg to set. + * @return This builder for chaining. + */ + public Builder setMsg( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + msg_ = value; + onChanged(); + return this; + } + /** + * 
+       * 采集响应信息
+       *
+ * + * string msg = 6; + * @return This builder for chaining. + */ + public Builder clearMsg() { + + msg_ = getDefaultInstance().getMsg(); + onChanged(); + return this; + } + /** + * 
+       * 采集响应信息
+       *
+ * + * string msg = 6; + * @param value The bytes for msg to set. + * @return This builder for chaining. + */ + public Builder setMsgBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + msg_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList fields_ = com.google.protobuf.LazyStringArrayList.EMPTY; + private void ensureFieldsIsMutable() { + if (!((bitField0_ & 0x00000001) != 0)) { + fields_ = new com.google.protobuf.LazyStringArrayList(fields_); + bitField0_ |= 0x00000001; + } + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @return A list containing the fields. + */ + public com.google.protobuf.ProtocolStringList + getFieldsList() { + return fields_.getUnmodifiableView(); + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @return The count of fields. + */ + public int getFieldsCount() { + return fields_.size(); + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @param index The index of the element to return. + * @return The fields at the given index. + */ + public java.lang.String getFields(int index) { + return fields_.get(index); + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @param index The index of the value to return. + * @return The bytes of the fields at the given index. + */ + public com.google.protobuf.ByteString + getFieldsBytes(int index) { + return fields_.getByteString(index); + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @param index The index to set the value at. + * @param value The fields to set. + * @return This builder for chaining. + */ + public Builder setFields( + int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureFieldsIsMutable(); + fields_.set(index, value); + onChanged(); + return this; + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @param value The fields to add. + * @return This builder for chaining. + */ + public Builder addFields( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureFieldsIsMutable(); + fields_.add(value); + onChanged(); + return this; + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @param values The fields to add. + * @return This builder for chaining. + */ + public Builder addAllFields( + java.lang.Iterable values) { + ensureFieldsIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, fields_); + onChanged(); + return this; + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @return This builder for chaining. + */ + public Builder clearFields() { + fields_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + /** + * 
+       * 采集指标名
+       *
+ * + * repeated string fields = 7; + * @param value The bytes of the fields to add. + * @return This builder for chaining. + */ + public Builder addFieldsBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + ensureFieldsIsMutable(); + fields_.add(value); + onChanged(); + return this; + } + + private java.util.List values_ = + java.util.Collections.emptyList(); + private void ensureValuesIsMutable() { + if (!((bitField0_ & 0x00000002) != 0)) { + values_ = new java.util.ArrayList(values_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + com.usthe.common.entity.message.CollectRep.ValueRow, com.usthe.common.entity.message.CollectRep.ValueRow.Builder, com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder> valuesBuilder_; + + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public java.util.List getValuesList() { + if (valuesBuilder_ == null) { + return java.util.Collections.unmodifiableList(values_); + } else { + return valuesBuilder_.getMessageList(); + } + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public int getValuesCount() { + if (valuesBuilder_ == null) { + return values_.size(); + } else { + return valuesBuilder_.getCount(); + } + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public com.usthe.common.entity.message.CollectRep.ValueRow getValues(int index) { + if (valuesBuilder_ == null) { + return values_.get(index); + } else { + return valuesBuilder_.getMessage(index); + } + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder setValues( + int index, com.usthe.common.entity.message.CollectRep.ValueRow value) { + if (valuesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.set(index, value); + onChanged(); + } else { + valuesBuilder_.setMessage(index, value); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder setValues( + int index, com.usthe.common.entity.message.CollectRep.ValueRow.Builder builderForValue) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.set(index, builderForValue.build()); + onChanged(); + } else { + valuesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder addValues(com.usthe.common.entity.message.CollectRep.ValueRow value) { + if (valuesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.add(value); + onChanged(); + } else { + valuesBuilder_.addMessage(value); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder addValues( + int index, com.usthe.common.entity.message.CollectRep.ValueRow value) { + if (valuesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.add(index, value); + onChanged(); + } else { + valuesBuilder_.addMessage(index, value); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder addValues( + com.usthe.common.entity.message.CollectRep.ValueRow.Builder builderForValue) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.add(builderForValue.build()); + onChanged(); + } else { + valuesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder addValues( + int index, com.usthe.common.entity.message.CollectRep.ValueRow.Builder builderForValue) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.add(index, builderForValue.build()); + onChanged(); + } else { + valuesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder addAllValues( + java.lang.Iterable values) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, values_); + onChanged(); + } else { + valuesBuilder_.addAllMessages(values); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder clearValues() { + if (valuesBuilder_ == null) { + values_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + valuesBuilder_.clear(); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public Builder removeValues(int index) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.remove(index); + onChanged(); + } else { + valuesBuilder_.remove(index); + } + return this; + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public com.usthe.common.entity.message.CollectRep.ValueRow.Builder getValuesBuilder( + int index) { + return getValuesFieldBuilder().getBuilder(index); + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder getValuesOrBuilder( + int index) { + if (valuesBuilder_ == null) { + return values_.get(index); } else { + return valuesBuilder_.getMessageOrBuilder(index); + } + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public java.util.List + getValuesOrBuilderList() { + if (valuesBuilder_ != null) { + return valuesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(values_); + } + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public com.usthe.common.entity.message.CollectRep.ValueRow.Builder addValuesBuilder() { + return getValuesFieldBuilder().addBuilder( + com.usthe.common.entity.message.CollectRep.ValueRow.getDefaultInstance()); + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public com.usthe.common.entity.message.CollectRep.ValueRow.Builder addValuesBuilder( + int index) { + return getValuesFieldBuilder().addBuilder( + index, com.usthe.common.entity.message.CollectRep.ValueRow.getDefaultInstance()); + } + /** + * 
+       * 采集指标值集合(fields作为字段名称与ValueRow映射)
+       *
+ * + * repeated .com.usthe.common.entity.message.ValueRow values = 8; + */ + public java.util.List + getValuesBuilderList() { + return getValuesFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilderV3< + com.usthe.common.entity.message.CollectRep.ValueRow, com.usthe.common.entity.message.CollectRep.ValueRow.Builder, com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder> + getValuesFieldBuilder() { + if (valuesBuilder_ == null) { + valuesBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< + com.usthe.common.entity.message.CollectRep.ValueRow, com.usthe.common.entity.message.CollectRep.ValueRow.Builder, com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder>( + values_, + ((bitField0_ & 0x00000002) != 0), + getParentForChildren(), + isClean()); + values_ = null; + } + return valuesBuilder_; + } + @java.lang.Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + @java.lang.Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + + // @@protoc_insertion_point(builder_scope:com.usthe.common.entity.message.MetricsData) + } + + // @@protoc_insertion_point(class_scope:com.usthe.common.entity.message.MetricsData) + private static final com.usthe.common.entity.message.CollectRep.MetricsData DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.usthe.common.entity.message.CollectRep.MetricsData(); + } + + public static com.usthe.common.entity.message.CollectRep.MetricsData getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + @java.lang.Override + public MetricsData parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new MetricsData(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.MetricsData getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ValueRowOrBuilder extends + // @@protoc_insertion_point(interface_extends:com.usthe.common.entity.message.ValueRow) + com.google.protobuf.MessageOrBuilder { + + /** + * 
+     * 主键实例，唯一标识这行数据
+     *
+ * + * string instance = 1; + * @return The instance. + */ + java.lang.String getInstance(); + /** + * 
+     * 主键实例，唯一标识这行数据
+     *
+ * + * string instance = 1; + * @return The bytes for instance. + */ + com.google.protobuf.ByteString + getInstanceBytes(); + + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @return A list containing the columns. + */ + java.util.List + getColumnsList(); + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @return The count of columns. + */ + int getColumnsCount(); + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @param index The index of the element to return. + * @return The columns at the given index. + */ + java.lang.String getColumns(int index); + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @param index The index of the value to return. + * @return The bytes of the columns at the given index. + */ + com.google.protobuf.ByteString + getColumnsBytes(int index); + } + /** + * Protobuf type {@code com.usthe.common.entity.message.ValueRow} + */ + public static final class ValueRow extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:com.usthe.common.entity.message.ValueRow) + ValueRowOrBuilder { + private static final long serialVersionUID = 0L; + // Use ValueRow.newBuilder() to construct. + private ValueRow(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private ValueRow() { + instance_ = ""; + columns_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance( + UnusedPrivateParameter unused) { + return new ValueRow(); + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private ValueRow( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 10: { + java.lang.String s = input.readStringRequireUtf8(); + + instance_ = s; + break; + } + case 18: { + java.lang.String s = input.readStringRequireUtf8(); + if (!((mutable_bitField0_ & 0x00000001) != 0)) { + columns_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + columns_.add(s); + break; + } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) != 0)) { + columns_ = columns_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_ValueRow_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_ValueRow_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.usthe.common.entity.message.CollectRep.ValueRow.class, com.usthe.common.entity.message.CollectRep.ValueRow.Builder.class); + } + + public static final int INSTANCE_FIELD_NUMBER = 1; + private volatile java.lang.Object instance_; + /** + * 
+     * 主键实例，唯一标识这行数据
+     *
+ * + * string instance = 1; + * @return The instance. + */ + @java.lang.Override + public java.lang.String getInstance() { + java.lang.Object ref = instance_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + instance_ = s; + return s; + } + } + /** + * 
+     * 主键实例，唯一标识这行数据
+     *
+ * + * string instance = 1; + * @return The bytes for instance. + */ + @java.lang.Override + public com.google.protobuf.ByteString + getInstanceBytes() { + java.lang.Object ref = instance_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + instance_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int COLUMNS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList columns_; + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @return A list containing the columns. + */ + public com.google.protobuf.ProtocolStringList + getColumnsList() { + return columns_; + } + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @return The count of columns. + */ + public int getColumnsCount() { + return columns_.size(); + } + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @param index The index of the element to return. + * @return The columns at the given index. + */ + public java.lang.String getColumns(int index) { + return columns_.get(index); + } + /** + * 
+     * 采集指标值
+     *
+ * + * repeated string columns = 2; + * @param index The index of the value to return. + * @return The bytes of the columns at the given index. + */ + public com.google.protobuf.ByteString + getColumnsBytes(int index) { + return columns_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + @java.lang.Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + @java.lang.Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (!getInstanceBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, instance_); + } + for (int i = 0; i < columns_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, columns_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + @java.lang.Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (!getInstanceBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, instance_); + } + { + int dataSize = 0; + for (int i = 0; i < columns_.size(); i++) { + dataSize += computeStringSizeNoTag(columns_.getRaw(i)); + } + size += dataSize; + size += 1 * getColumnsList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.usthe.common.entity.message.CollectRep.ValueRow)) { + return super.equals(obj); + } + com.usthe.common.entity.message.CollectRep.ValueRow other = (com.usthe.common.entity.message.CollectRep.ValueRow) obj; + + if (!getInstance() + .equals(other.getInstance())) return false; + if (!getColumnsList() + .equals(other.getColumnsList())) return false; + if (!unknownFields.equals(other.unknownFields)) return false; + return true; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + INSTANCE_FIELD_NUMBER; + hash = (53 * hash) + getInstance().hashCode(); + if (getColumnsCount() > 0) { + hash = (37 * hash) + COLUMNS_FIELD_NUMBER; + hash = (53 * hash) + getColumnsList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static com.usthe.common.entity.message.CollectRep.ValueRow parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @java.lang.Override + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(com.usthe.common.entity.message.CollectRep.ValueRow prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + @java.lang.Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code com.usthe.common.entity.message.ValueRow} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:com.usthe.common.entity.message.ValueRow) + com.usthe.common.entity.message.CollectRep.ValueRowOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_ValueRow_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_ValueRow_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.usthe.common.entity.message.CollectRep.ValueRow.class, com.usthe.common.entity.message.CollectRep.ValueRow.Builder.class); + } + + // Construct using com.usthe.common.entity.message.CollectRep.ValueRow.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + @java.lang.Override + public Builder clear() { + super.clear(); + instance_ = ""; + + columns_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return com.usthe.common.entity.message.CollectRep.internal_static_com_usthe_common_entity_message_ValueRow_descriptor; + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.ValueRow getDefaultInstanceForType() { + return com.usthe.common.entity.message.CollectRep.ValueRow.getDefaultInstance(); + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.ValueRow build() { + com.usthe.common.entity.message.CollectRep.ValueRow result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.ValueRow buildPartial() { + com.usthe.common.entity.message.CollectRep.ValueRow result = new com.usthe.common.entity.message.CollectRep.ValueRow(this); + int from_bitField0_ = bitField0_; + result.instance_ = instance_; + if (((bitField0_ & 0x00000001) != 0)) { + columns_ = columns_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.columns_ = columns_; + onBuilt(); + return result; + } + + @java.lang.Override + public Builder clone() { + return super.clone(); + } + @java.lang.Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return super.setField(field, value); + } + @java.lang.Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return super.clearField(field); + } + @java.lang.Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return super.clearOneof(oneof); + } + @java.lang.Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, java.lang.Object value) { + return super.setRepeatedField(field, index, value); + } + @java.lang.Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return super.addRepeatedField(field, value); + } + @java.lang.Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.usthe.common.entity.message.CollectRep.ValueRow) { + return mergeFrom((com.usthe.common.entity.message.CollectRep.ValueRow)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.usthe.common.entity.message.CollectRep.ValueRow other) { + if (other == com.usthe.common.entity.message.CollectRep.ValueRow.getDefaultInstance()) return this; + if (!other.getInstance().isEmpty()) { + instance_ = other.instance_; + onChanged(); + } + if (!other.columns_.isEmpty()) { + if (columns_.isEmpty()) { + columns_ = other.columns_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureColumnsIsMutable(); + columns_.addAll(other.columns_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @java.lang.Override + public final boolean isInitialized() { + return true; + } + + @java.lang.Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.usthe.common.entity.message.CollectRep.ValueRow parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.usthe.common.entity.message.CollectRep.ValueRow) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + private java.lang.Object instance_ = ""; + /** + * 
+       * 主键实例，唯一标识这行数据
+       *
+ * + * string instance = 1; + * @return The instance. + */ + public java.lang.String getInstance() { + java.lang.Object ref = instance_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + instance_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * 
+       * 主键实例，唯一标识这行数据
+       *
+ * + * string instance = 1; + * @return The bytes for instance. + */ + public com.google.protobuf.ByteString + getInstanceBytes() { + java.lang.Object ref = instance_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + instance_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * 
+       * 主键实例，唯一标识这行数据
+       *
+ * + * string instance = 1; + * @param value The instance to set. + * @return This builder for chaining. + */ + public Builder setInstance( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + instance_ = value; + onChanged(); + return this; + } + /** + * 
+       * 主键实例，唯一标识这行数据
+       *
+ * + * string instance = 1; + * @return This builder for chaining. + */ + public Builder clearInstance() { + + instance_ = getDefaultInstance().getInstance(); + onChanged(); + return this; + } + /** + * 
+       * 主键实例，唯一标识这行数据
+       *
+ * + * string instance = 1; + * @param value The bytes for instance to set. + * @return This builder for chaining. + */ + public Builder setInstanceBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + instance_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList columns_ = com.google.protobuf.LazyStringArrayList.EMPTY; + private void ensureColumnsIsMutable() { + if (!((bitField0_ & 0x00000001) != 0)) { + columns_ = new com.google.protobuf.LazyStringArrayList(columns_); + bitField0_ |= 0x00000001; + } + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @return A list containing the columns. + */ + public com.google.protobuf.ProtocolStringList + getColumnsList() { + return columns_.getUnmodifiableView(); + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @return The count of columns. + */ + public int getColumnsCount() { + return columns_.size(); + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @param index The index of the element to return. + * @return The columns at the given index. + */ + public java.lang.String getColumns(int index) { + return columns_.get(index); + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @param index The index of the value to return. + * @return The bytes of the columns at the given index. + */ + public com.google.protobuf.ByteString + getColumnsBytes(int index) { + return columns_.getByteString(index); + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @param index The index to set the value at. + * @param value The columns to set. + * @return This builder for chaining. + */ + public Builder setColumns( + int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.set(index, value); + onChanged(); + return this; + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @param value The columns to add. + * @return This builder for chaining. + */ + public Builder addColumns( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.add(value); + onChanged(); + return this; + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @param values The columns to add. + * @return This builder for chaining. + */ + public Builder addAllColumns( + java.lang.Iterable values) { + ensureColumnsIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, columns_); + onChanged(); + return this; + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @return This builder for chaining. + */ + public Builder clearColumns() { + columns_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + /** + * 
+       * 采集指标值
+       *
+ * + * repeated string columns = 2; + * @param value The bytes of the columns to add. + * @return This builder for chaining. + */ + public Builder addColumnsBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + ensureColumnsIsMutable(); + columns_.add(value); + onChanged(); + return this; + } + @java.lang.Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + @java.lang.Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + + // @@protoc_insertion_point(builder_scope:com.usthe.common.entity.message.ValueRow) + } + + // @@protoc_insertion_point(class_scope:com.usthe.common.entity.message.ValueRow) + private static final com.usthe.common.entity.message.CollectRep.ValueRow DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.usthe.common.entity.message.CollectRep.ValueRow(); + } + + public static com.usthe.common.entity.message.CollectRep.ValueRow getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + @java.lang.Override + public ValueRow parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ValueRow(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + @java.lang.Override + public com.usthe.common.entity.message.CollectRep.ValueRow getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_usthe_common_entity_message_MetricsData_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_usthe_common_entity_message_MetricsData_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_usthe_common_entity_message_ValueRow_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_usthe_common_entity_message_ValueRow_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\021collect_rep.proto\022\037com.usthe.common.en" + + "tity.message\"\322\001\n\013MetricsData\022\n\n\002id\030\001 \001(\004" + + "\022\013\n\003app\030\002 \001(\t\022\017\n\007metrics\030\003 \001(\t\022\014\n\004time\030\004" + + " \001(\004\0223\n\004code\030\005 \001(\0162%.com.usthe.common.en" + + "tity.message.Code\022\013\n\003msg\030\006 \001(\t\022\016\n\006fields" + + "\030\007 \003(\t\0229\n\006values\030\010 \003(\0132).com.usthe.commo" + + "n.entity.message.ValueRow\"-\n\010ValueRow\022\020\n" + + "\010instance\030\001 \001(\t\022\017\n\007columns\030\002 \003(\t*b\n\004Code" + + "\022\013\n\007SUCCESS\020\000\022\020\n\014UN_AVAILABLE\020\001\022\020\n\014UN_RE" + + "ACHABLE\020\002\022\022\n\016UN_CONNECTABLE\020\003\022\010\n\004FAIL\020\004\022" + + "\013\n\007TIMEOUT\020\005b\006proto3" + }; + descriptor = com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }); + internal_static_com_usthe_common_entity_message_MetricsData_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_com_usthe_common_entity_message_MetricsData_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_usthe_common_entity_message_MetricsData_descriptor, + new java.lang.String[] { "Id", "App", "Metrics", "Time", "Code", "Msg", "Fields", "Values", }); + internal_static_com_usthe_common_entity_message_ValueRow_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_com_usthe_common_entity_message_ValueRow_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_usthe_common_entity_message_ValueRow_descriptor, + new java.lang.String[] { "Instance", "Columns", }); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/common/src/main/java/com/usthe/common/util/GsonUtil.java b/common/src/main/java/com/usthe/common/util/GsonUtil.java new file mode 100644 index 0000000..4508a84 --- /dev/null +++ b/common/src/main/java/com/usthe/common/util/GsonUtil.java @@ -0,0 +1,41 @@ +package com.usthe.common.util; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.etcd.jetcd.ByteSequence; + +import javax.annotation.concurrent.ThreadSafe; +import java.nio.charset.StandardCharsets; + +/** + * gson 工具类 + * @author tomsun28 + * @date 2021/10/16 20:49 + */ +@ThreadSafe +public class GsonUtil { + + private static Gson gson; + + static { + gson = new GsonBuilder().enableComplexMapKeySerialization() + .serializeNulls() + .create(); + } + + public static String toJson(Object source) { + return gson.toJson(source); + } + + public static T fromJson(String jsonStr, Class clazz) { + return gson.fromJson(jsonStr, clazz); + } + + public static T fromJson(ByteSequence byteSequence, Class clazz) { + if (byteSequence == null || byteSequence.isEmpty()) { + return null; + } + return gson.fromJson(byteSequence.toString(StandardCharsets.UTF_8), clazz); + } + +} diff --git a/common/src/main/message/collect_rep.proto b/common/src/main/message/collect_rep.proto new file mode 100644 index 0000000..0b9c5fb --- /dev/null +++ b/common/src/main/message/collect_rep.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; +package com.usthe.common.entity.message; + +message MetricsData +{ + // 监控的ID + uint64 id = 1; + // 监控的类型 eg: linux | mysql | jvm + string app = 2; + // 监控采集的指标集合 eg: cpu | memory | health + string metrics = 3; + // 采集时间 + uint64 time = 4; + // 采集响应码 + Code code = 5; + // 采集响应信息 + string msg = 6; + // 采集指标名 + repeated string fields = 7; + // 采集指标值集合(fields作为字段名称与ValueRow映射) + repeated ValueRow values = 8; +} + +message ValueRow +{ + // 主键实例，唯一标识这行数据 + string instance = 1; + // 采集指标值 + repeated string columns = 2; +} + +enum Code +{ + // 采集成功 + SUCCESS = 0; + // 采集器不可用 + UN_AVAILABLE = 1; + // 对端不可达(网络层icmp) + UN_REACHABLE = 2; + // 对端连接失败(传输层tcp,udp) + UN_CONNECTABLE = 3; + // 数据采集失败(应用层http,ssh,snmp) + FAIL = 4; + // 采集超时 + TIMEOUT = 5; +} \ No newline at end of file diff --git a/manager/pom.xml b/manager/pom.xml new file mode 100644 index 0000000..0bf1bcb --- /dev/null +++ b/manager/pom.xml @@ -0,0 +1,18 @@ + + + + monitor + com.usthe.tancloud + 1.0-SNAPSHOT + + 4.0.0 + + manager + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..99dc15d --- /dev/null +++ b/pom.xml @@ -0,0 +1,129 @@ + + + 4.0.0 + + com.usthe.tancloud + monitor + pom + 1.0-SNAPSHOT + + scheduler + manager + alerter + common + collector + + + + 1.8 + 1.8 + 1.8 + UTF-8 + 1.7.21 + 2.3.0 + 1.18.20 + + 5.7.0 + 4.0.2 + + + + + + + org.springframework.boot + spring-boot-dependencies + 2.4.5 + pom + import + + + javax.xml.bind + jaxb-api + ${xml.bind.version} + + + + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.projectlombok + lombok + ${lombok.version} + provided + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.easymock + easymock + ${easymock.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.8 + + + rulesets/java/ali-comment.xml + rulesets/java/ali-concurrent.xml + rulesets/java/ali-constant.xml + rulesets/java/ali-exception.xml + + rulesets/java/ali-naming.xml + rulesets/java/ali-oop.xml + rulesets/java/ali-orm.xml + rulesets/java/ali-other.xml + rulesets/java/ali-set.xml + + true + + + + validate + validate + + check + + + + + + com.alibaba.p3c + p3c-pmd + 1.3.0 + + + + + + \ No newline at end of file