mirror of
https://github.com/didi/KnowStreaming.git
synced 2025-12-24 11:52:08 +08:00
Merge branch 'dev' of github.com:kingdomrushing/LogiKM into dev_v2.6.0
This commit is contained in:
@@ -15,6 +15,7 @@ server:
|
||||
spring:
|
||||
application:
|
||||
name: kafkamanager
|
||||
version: @project.version@
|
||||
profiles:
|
||||
active: dev
|
||||
datasource:
|
||||
|
||||
@@ -15,6 +15,7 @@ server:
|
||||
spring:
|
||||
application:
|
||||
name: kafkamanager
|
||||
version: @project.version@
|
||||
profiles:
|
||||
active: dev
|
||||
datasource:
|
||||
@@ -26,7 +27,6 @@ spring:
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
|
||||
|
||||
servlet:
|
||||
multipart:
|
||||
max-file-size: 100MB
|
||||
@@ -36,28 +36,32 @@ logging:
|
||||
config: classpath:logback-spring.xml
|
||||
|
||||
custom:
|
||||
idc: cn # 部署的数据中心, 忽略该配置, 后续会进行删除
|
||||
jmx:
|
||||
max-conn: 10 # 2.3版本配置不在这个地方生效
|
||||
idc: cn
|
||||
store-metrics-task:
|
||||
community:
|
||||
broker-metrics-enabled: true # 社区部分broker metrics信息收集开关, 关闭之后metrics信息将不会进行收集及写DB
|
||||
topic-metrics-enabled: true # 社区部分topic的metrics信息收集开关, 关闭之后metrics信息将不会进行收集及写DB
|
||||
didi:
|
||||
app-topic-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标,因此默认关闭
|
||||
topic-request-time-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标,因此默认关闭
|
||||
topic-throttled-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标,因此默认关闭
|
||||
topic-metrics-enabled: true
|
||||
didi: # 滴滴Kafka特有的指标
|
||||
app-topic-metrics-enabled: false
|
||||
topic-request-time-metrics-enabled: false
|
||||
topic-throttled-metrics-enabled: false
|
||||
|
||||
# 任务相关的开关
|
||||
# 任务相关的配置
|
||||
task:
|
||||
op:
|
||||
sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
|
||||
order-auto-exec: # 工单自动化审批线程的开关
|
||||
topic-enabled: false # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
app-enabled: false # App工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
|
||||
order-auto-exec: # 工单自动化审批线程的开关
|
||||
topic-enabled: false # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
app-enabled: false # App工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
metrics:
|
||||
delete-metrics:
|
||||
delete-limit-size: 1000
|
||||
collect: # 收集指标
|
||||
broker-metrics-enabled: true # 收集Broker指标
|
||||
sink: # 上报指标
|
||||
cluster-metrics: # 上报cluster指标
|
||||
sink-db-enabled: true # 上报到db
|
||||
broker-metrics: # 上报broker指标
|
||||
sink-db-enabled: true # 上报到db
|
||||
delete: # 删除指标
|
||||
delete-limit-size: 1000 # 单次删除的批大小
|
||||
cluster-metrics-save-days: 14 # 集群指标保存天数
|
||||
broker-metrics-save-days: 14 # Broker指标保存天数
|
||||
topic-metrics-save-days: 7 # Topic指标保存天数
|
||||
@@ -65,64 +69,6 @@ task:
|
||||
topic-throttled-metrics-save-days: 7 # Topic限流指标保存天数
|
||||
app-topic-metrics-save-days: 7 # App+Topic指标保存天数
|
||||
|
||||
# ldap相关的配置
|
||||
account:
|
||||
ldap:
|
||||
enabled: false
|
||||
url: ldap://127.0.0.1:389/
|
||||
basedn: dc=tsign,dc=cn
|
||||
factory: com.sun.jndi.ldap.LdapCtxFactory
|
||||
filter: sAMAccountName
|
||||
security:
|
||||
authentication: simple
|
||||
principal: cn=admin,dc=tsign,dc=cn
|
||||
credentials: admin
|
||||
auth-user-registration: true
|
||||
auth-user-registration-role: normal
|
||||
|
||||
# 集群升级部署相关的功能,需要配合夜莺及S3进行使用
|
||||
kcm:
|
||||
enabled: false
|
||||
s3:
|
||||
endpoint: s3.didiyunapi.com
|
||||
access-key: 1234567890
|
||||
secret-key: 0987654321
|
||||
bucket: logi-kafka
|
||||
n9e:
|
||||
base-url: http://127.0.0.1:8004
|
||||
user-token: 12345678
|
||||
timeout: 300
|
||||
account: root
|
||||
script-file: kcm_script.sh
|
||||
|
||||
# 监控告警相关的功能,需要配合夜莺进行使用
|
||||
# enabled: 表示是否开启监控告警的功能, true: 开启, false: 不开启
|
||||
# n9e.nid: 夜莺的节点ID
|
||||
# n9e.user-token: 用户的密钥,在夜莺的个人设置中
|
||||
# n9e.mon.base-url: 监控地址
|
||||
# n9e.sink.base-url: 数据上报地址
|
||||
# n9e.rdb.base-url: 用户资源中心地址
|
||||
|
||||
monitor:
|
||||
enabled: false
|
||||
n9e:
|
||||
nid: 2
|
||||
user-token: 1234567890
|
||||
mon:
|
||||
base-url: http://127.0.0.1:8000 # 夜莺v4版本,默认端口统一调整为了8000
|
||||
sink:
|
||||
base-url: http://127.0.0.1:8000 # 夜莺v4版本,默认端口统一调整为了8000
|
||||
rdb:
|
||||
base-url: http://127.0.0.1:8000 # 夜莺v4版本,默认端口统一调整为了8000
|
||||
|
||||
|
||||
notify: # 通知的功能
|
||||
kafka: # 默认通知发送到kafka的指定Topic中
|
||||
cluster-id: 95 # Topic的集群ID
|
||||
topic-name: didi-kafka-notify # Topic名称
|
||||
order: # 部署的KM的地址
|
||||
detail-url: http://127.0.0.1
|
||||
|
||||
thread-pool:
|
||||
collect-metrics:
|
||||
thread-num: 256 # 收集指标线程池大小
|
||||
@@ -136,4 +82,55 @@ client-pool:
|
||||
min-idle-client-num: 24 # 最小空闲客户端数
|
||||
max-idle-client-num: 24 # 最大空闲客户端数
|
||||
max-total-client-num: 24 # 最大客户端数
|
||||
borrow-timeout-unit-ms: 3000 # 租借超时时间,单位秒
|
||||
borrow-timeout-unit-ms: 3000 # 租借超时时间,单位毫秒
|
||||
|
||||
account:
|
||||
jump-login:
|
||||
gateway-api: false # 网关接口
|
||||
third-part-api: false # 第三方接口
|
||||
ldap:
|
||||
enabled: false
|
||||
url: ldap://127.0.0.1:389/
|
||||
basedn: dc=tsign,dc=cn
|
||||
factory: com.sun.jndi.ldap.LdapCtxFactory
|
||||
filter: sAMAccountName
|
||||
security:
|
||||
authentication: simple
|
||||
principal: cn=admin,dc=tsign,dc=cn
|
||||
credentials: admin
|
||||
auth-user-registration: true
|
||||
auth-user-registration-role: normal
|
||||
|
||||
kcm: # 集群安装部署,仅安装broker
|
||||
enabled: false # 是否开启
|
||||
s3: # s3 存储服务
|
||||
endpoint: s3.didiyunapi.com
|
||||
access-key: 1234567890
|
||||
secret-key: 0987654321
|
||||
bucket: logi-kafka
|
||||
n9e: # 夜莺
|
||||
base-url: http://127.0.0.1:8004 # 夜莺job服务地址
|
||||
user-token: 12345678 # 用户的token
|
||||
timeout: 300 # 当台操作的超时时间
|
||||
account: root # 操作时使用的账号
|
||||
script-file: kcm_script.sh # 脚本,已内置好,在源码的kcm模块内,此处配置无需修改
|
||||
logikm-url: http://127.0.0.1:8080 # logikm部署地址,部署时kcm_script.sh会调用logikm检查部署中的一些状态
|
||||
|
||||
monitor:
|
||||
enabled: false
|
||||
n9e:
|
||||
nid: 2
|
||||
user-token: 1234567890
|
||||
mon:
|
||||
base-url: http://127.0.0.1:8000 # 夜莺v4版本,默认端口统一调整为了8000
|
||||
sink:
|
||||
base-url: http://127.0.0.1:8000 # 夜莺v4版本,默认端口统一调整为了8000
|
||||
rdb:
|
||||
base-url: http://127.0.0.1:8000 # 夜莺v4版本,默认端口统一调整为了8000
|
||||
|
||||
notify:
|
||||
kafka:
|
||||
cluster-id: 95
|
||||
topic-name: didi-kafka-notify
|
||||
order:
|
||||
detail-url: http://127.0.0.1
|
||||
|
||||
@@ -13,6 +13,9 @@ CREATE TABLE `account` (
|
||||
`username` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '用户名',
|
||||
`password` varchar(128) NOT NULL DEFAULT '' COMMENT '密码',
|
||||
`role` tinyint(8) NOT NULL DEFAULT '0' COMMENT '角色类型, 0:普通用户 1:研发 2:运维',
|
||||
`department` varchar(256) DEFAULT '' COMMENT '部门名',
|
||||
`display_name` varchar(256) DEFAULT '' COMMENT '用户姓名',
|
||||
`mail` varchar(256) DEFAULT '' COMMENT '邮箱',
|
||||
`status` int(16) NOT NULL DEFAULT '0' COMMENT '0标识使用中,-1标识已废弃',
|
||||
`gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`gmt_modify` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||||
|
||||
@@ -39,4 +39,14 @@ ALTER TABLE `gateway_config`
|
||||
ADD COLUMN `description` TEXT NULL COMMENT '描述信息' AFTER `version`;
|
||||
```
|
||||
|
||||
### 升级至`2.6.0`版本
|
||||
|
||||
#### 1.mysql变更
|
||||
`2.6.0`版本在`account`表增加用户姓名,部门名,邮箱三个字段,因此需要执行下面的sql进行字段的增加。
|
||||
|
||||
```sql
|
||||
ALTER TABLE `account`
|
||||
ADD COLUMN `display_name` VARCHAR(256) NOT NULL DEFAULT '' COMMENT '用户名' AFTER `role`,
|
||||
ADD COLUMN `department` VARCHAR(256) NOT NULL DEFAULT '' COMMENT '部门名' AFTER `display_name`,
|
||||
ADD COLUMN `mail` VARCHAR(256) NOT NULL DEFAULT '' COMMENT '邮箱' AFTER `department`;
|
||||
```
|
||||
|
||||
BIN
docs/dev_guide/assets/kcm/kcm_principle.png
Normal file
BIN
docs/dev_guide/assets/kcm/kcm_principle.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 69 KiB |
89
docs/dev_guide/drawio/KCM实现原理.drawio
Normal file
89
docs/dev_guide/drawio/KCM实现原理.drawio
Normal file
@@ -0,0 +1,89 @@
|
||||
<mxfile host="65bd71144e">
|
||||
<diagram id="bhaMuW99Q1BzDTtcfRXp" name="Page-1">
|
||||
<mxGraphModel dx="1138" dy="830" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0"/>
|
||||
<mxCell id="1" parent="0"/>
|
||||
<mxCell id="11" value="待部署Kafka-Broker的机器" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;labelPosition=center;verticalLabelPosition=bottom;align=center;verticalAlign=top;dashed=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="380" y="240" width="320" height="240" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="24" value="" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;labelPosition=center;verticalLabelPosition=bottom;align=center;verticalAlign=top;dashed=1;fillColor=#eeeeee;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="410" y="310" width="260" height="160" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="6" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="2" target="3">
|
||||
<mxGeometry relative="1" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="7" value="调用夜莺接口,<br>创建集群安装部署任务" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="6">
|
||||
<mxGeometry x="-0.0875" y="1" relative="1" as="geometry">
|
||||
<mxPoint x="9" y="1" as="offset"/>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="9" style="edgeStyle=none;html=1;" edge="1" parent="1" source="2" target="4">
|
||||
<mxGeometry relative="1" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="10" value="通过版本管理,将Kafka的安装包,<br>server配置上传到s3中" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="9">
|
||||
<mxGeometry x="0.0125" y="2" relative="1" as="geometry">
|
||||
<mxPoint as="offset"/>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="2" value="LogiKM" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1">
|
||||
<mxGeometry x="40" y="100" width="120" height="40" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="12" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="3" target="5">
|
||||
<mxGeometry relative="1" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="13" value="1、下发任务脚本(kcm_script.sh);<br>2、下发任务操作命令;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="12">
|
||||
<mxGeometry x="-0.0731" y="2" relative="1" as="geometry">
|
||||
<mxPoint x="-2" y="-16" as="offset"/>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="3" value="夜莺——任务中心" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#cdeb8b;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="480" y="100" width="120" height="40" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="4" value="S3" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
|
||||
<mxGeometry x="40" y="310" width="120" height="40" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="5" value="夜莺——Agent(<font color="#ff3333">代理执行kcm_script.sh脚本</font>)" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
|
||||
<mxGeometry x="400" y="260" width="280" height="40" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="22" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;fontColor=#FF3333;exitX=0;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="14" target="4">
|
||||
<mxGeometry relative="1" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="25" value="下载安装包" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontColor=#000000;" vertex="1" connectable="0" parent="22">
|
||||
<mxGeometry x="0.2226" y="-2" relative="1" as="geometry">
|
||||
<mxPoint x="27" y="2" as="offset"/>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="14" value="执行kcm_script.sh脚本:下载安装包" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#eeeeee;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="425" y="320" width="235" height="20" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="18" value="执行kcm_script.sh脚本:安装" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#eeeeee;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="425" y="350" width="235" height="20" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="19" value="执行kcm_script.sh脚本:检查安装结果" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#eeeeee;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="425" y="380" width="235" height="20" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="23" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fontColor=#FF3333;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="20" target="2">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="770" y="420"/>
|
||||
<mxPoint x="770" y="40"/>
|
||||
<mxPoint x="100" y="40"/>
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="26" value="检查副本同步状态" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontColor=#000000;" vertex="1" connectable="0" parent="23">
|
||||
<mxGeometry x="-0.3344" relative="1" as="geometry">
|
||||
<mxPoint as="offset"/>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="20" value="执行kcm_script.sh脚本:检查副本同步" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#eeeeee;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="425" y="410" width="235" height="20" as="geometry"/>
|
||||
</mxCell>
|
||||
<mxCell id="21" value="执行kcm_script.sh脚本:结束" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=1;fillColor=#eeeeee;strokeColor=#36393d;" vertex="1" parent="1">
|
||||
<mxGeometry x="425" y="440" width="235" height="20" as="geometry"/>
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
@@ -1,27 +0,0 @@
|
||||
|
||||
---
|
||||
|
||||

|
||||
|
||||
**一站式`Apache Kafka`集群指标监控与运维管控平台**
|
||||
|
||||
---
|
||||
|
||||
# 升级至`2.2.0`版本
|
||||
|
||||
`2.2.0`版本在`cluster`表及`logical_cluster`各增加了一个字段,因此需要执行下面的sql进行字段的增加。
|
||||
|
||||
```sql
|
||||
# 往cluster表中增加jmx_properties字段, 这个字段会用于存储jmx相关的认证以及配置信息
|
||||
ALTER TABLE `cluster` ADD COLUMN `jmx_properties` TEXT NULL COMMENT 'JMX配置' AFTER `security_properties`;
|
||||
|
||||
# 往logical_cluster中增加identification字段, 同时数据和原先name数据相同, 最后增加一个唯一键.
|
||||
# 此后, name字段还是表示集群名称, 而identification字段表示的是集群标识, 只能是字母数字及下划线组成,
|
||||
# 数据上报到监控系统时, 集群这个标识采用的字段就是identification字段, 之前使用的是name字段.
|
||||
ALTER TABLE `logical_cluster` ADD COLUMN `identification` VARCHAR(192) NOT NULL DEFAULT '' COMMENT '逻辑集群标识' AFTER `name`;
|
||||
|
||||
UPDATE `logical_cluster` SET `identification`=`name` WHERE id>=0;
|
||||
|
||||
ALTER TABLE `logical_cluster` ADD INDEX `uniq_identification` (`identification` ASC);
|
||||
```
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
|
||||
---
|
||||
|
||||

|
||||
|
||||
**一站式`Apache Kafka`集群指标监控与运维管控平台**
|
||||
|
||||
---
|
||||
|
||||
# 升级至`2.3.0`版本
|
||||
|
||||
`2.3.0`版本在`gateway_config`表增加了一个描述说明的字段,因此需要执行下面的sql进行字段的增加。
|
||||
|
||||
```sql
|
||||
ALTER TABLE `gateway_config`
|
||||
ADD COLUMN `description` TEXT NULL COMMENT '描述信息' AFTER `version`;
|
||||
```
|
||||
89
docs/dev_guide/如何使用集群安装部署功能.md
Normal file
89
docs/dev_guide/如何使用集群安装部署功能.md
Normal file
@@ -0,0 +1,89 @@
|
||||
|
||||
---
|
||||
|
||||

|
||||
|
||||
**一站式`Apache Kafka`集群指标监控与运维管控平台**
|
||||
|
||||
---
|
||||
|
||||
# 如何使用集群安装部署功能?
|
||||
|
||||
[TOC]
|
||||
|
||||
## 1、实现原理
|
||||
|
||||

|
||||
|
||||
- LogiKM上传安装包到S3服务;
|
||||
- LogiKM调用夜莺-Job服务接口,创建执行[kcm_script.sh](https://github.com/didi/LogiKM/blob/master/kafka-manager-extends/kafka-manager-kcm/src/main/resources/kcm_script.sh)脚本的任务,kcm_script.sh脚本是安装部署Kafka集群的脚本;
|
||||
- 夜莺将任务脚本下发到具体的机器上,通过夜莺Agent执行该脚本;
|
||||
- kcm_script.sh脚本会进行Kafka-Broker的安装部署;
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 2、使用方式
|
||||
|
||||
### 2.1、第一步:修改配置
|
||||
|
||||
**配置application.yml文件**
|
||||
```yaml
|
||||
#
|
||||
kcm:
|
||||
enabled: false # 是否开启,将其修改为true
|
||||
s3: # s3 存储服务
|
||||
endpoint: s3.didiyunapi.com
|
||||
access-key: 1234567890
|
||||
secret-key: 0987654321
|
||||
bucket: logi-kafka
|
||||
n9e: # 夜莺
|
||||
base-url: http://127.0.0.1:8004 # 夜莺job服务地址
|
||||
user-token: 12345678 # 用户的token
|
||||
timeout: 300 # 单台操作的超时时间
|
||||
account: root # 操作时使用的账号
|
||||
script-file: kcm_script.sh # 脚本,已内置好,在源码的kcm模块内,此处配置无需修改
|
||||
logikm-url: http://127.0.0.1:8080 # logikm部署地址,部署时kcm_script.sh会调用logikm检查部署中的一些状态,这里只需要填写 http://IP:PORT 就可以了
|
||||
|
||||
|
||||
account:
|
||||
jump-login:
|
||||
gateway-api: false # 网关接口
|
||||
third-part-api: false # 第三方接口,将其修改为true,即允许未登录情况下调用开放的第三方接口
|
||||
```
|
||||
|
||||
### 2.2、第二步:检查服务
|
||||
|
||||
**检查s3服务**
|
||||
- 测试 "运维管控-》集群运维-》版本管理" 页面的上传,查看等功能是否都正常。如果存在不正常,则需要查看s3的配置是否正确;
|
||||
- 如果都没有问题,则上传Kafka的以.tgz结尾的安装包以及server.properties文件;
|
||||
|
||||
**检查夜莺Job服务**
|
||||
- 创建一个job任务,机器选择需要安装Kafka集群的机器,然后执行的命令是echo "Hello LogiKM",看能否被成功执行。如果不行,则需要检查夜莺的安装;
|
||||
- 如果没有问题则表示夜莺和所需部署的机器之间的交互是没有问题的;
|
||||
|
||||
### 2.3、第三步:接入集群
|
||||
|
||||
在LogiKM的 “运维管控-》集群列表” 中接入需要安装部署的集群,**PS:此时是允许接入一个没有任何Broker的空的Kafka集群**,其中对的bootstrapServers配置搭建完成后的Kafka集群地址就可以了,而ZK地址必须和集群的server.properties中的ZK地址保持一致;
|
||||
|
||||
### 2.4、第四步:部署集群
|
||||
|
||||
- 打开LogiKM的 “运维管控-》集群运维-》集群任务” 页面,点击 “新建集群任务” 按钮;
|
||||
- 选择集群、任务类型、包版本、server配置及填写主机列表,然后点击确认,即可在夜莺的Job服务中心中创建一个任务出来。**PS:如果创建失败,可以看一下日志我为什么创建失败**;
|
||||
- 随后可以点击详情及状态对任务进行操作;
|
||||
|
||||
### 2.5、可能问题
|
||||
|
||||
#### 2.5.1、问题一:任务执行超时、失败等
|
||||
|
||||
进入夜莺Job服务中心,查看对应的任务的相关日志;
|
||||
|
||||
- 提示安装包下载失败,则需要查看对应的s3服务是否可以直接wget下载安装包,如果不可以则需要对kcm_script.sh脚本进行修改;
|
||||
- 提示调用LogiKM失败,则可以使用postman手动测试一下kcm_script.sh脚本调用LogiKM的那个接口是否有问题,如果存在问题则进行相应的修改;PS:具体接口见kcm_script.sh脚本
|
||||
|
||||
|
||||
## 3、备注说明
|
||||
|
||||
- 集群安装部署,仅安装部署Kafka-Broker,不安装Kafka的ZK服务;
|
||||
- 安装部署中,有任何定制化的需求,例如修改安装的目录等,可以通过修改kcm_script.sh脚本实现;
|
||||
- kcm_script.sh脚本位置:[kcm_script.sh](https://github.com/didi/LogiKM/blob/master/kafka-manager-extends/kafka-manager-kcm/src/main/resources/kcm_script.sh);
|
||||
@@ -21,15 +21,12 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-web</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- http -->
|
||||
|
||||
@@ -20,12 +20,6 @@ public class ApiPrefix {
|
||||
// open
|
||||
public static final String API_V1_THIRD_PART_PREFIX = API_V1_PREFIX + "third-part/";
|
||||
|
||||
// 开放给OP的接口, 后续对 应的接口的集群都需要是物理集群
|
||||
public static final String API_V1_THIRD_PART_OP_PREFIX = API_V1_THIRD_PART_PREFIX + "op/";
|
||||
|
||||
// 开放给Normal的接口, 后续对应的接口的集群,都需要是逻辑集群
|
||||
public static final String API_V1_THIRD_PART_NORMAL_PREFIX = API_V1_THIRD_PART_PREFIX + "normal/";
|
||||
|
||||
// gateway
|
||||
public static final String GATEWAY_API_V1_PREFIX = "/gateway" + API_V1_PREFIX;
|
||||
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
package com.xiaojukeji.kafka.manager.common.constant;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 20/8/10
|
||||
*/
|
||||
public class LogConstant {
|
||||
public static final String COLLECTOR_METRICS_LOGGER = "COLLECTOR_METRICS_LOGGER";
|
||||
|
||||
public static final String API_METRICS_LOGGER = "API_METRICS_LOGGER";
|
||||
|
||||
public static final String SCHEDULED_TASK_LOGGER = "SCHEDULED_TASK_LOGGER";
|
||||
|
||||
private LogConstant() {
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,15 @@ public class AccountDTO {
|
||||
@ApiModelProperty(value = "角色")
|
||||
private Integer role;
|
||||
|
||||
@ApiModelProperty(value = "用户姓名")
|
||||
private String displayName;
|
||||
|
||||
@ApiModelProperty(value = "部门")
|
||||
private String department;
|
||||
|
||||
@ApiModelProperty(value = "邮箱")
|
||||
private String mail;
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
@@ -45,12 +54,39 @@ public class AccountDTO {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
public String getDisplayName() {
|
||||
return displayName;
|
||||
}
|
||||
|
||||
public void setDisplayName(String displayName) {
|
||||
this.displayName = displayName;
|
||||
}
|
||||
|
||||
public String getDepartment() {
|
||||
return department;
|
||||
}
|
||||
|
||||
public void setDepartment(String department) {
|
||||
this.department = department;
|
||||
}
|
||||
|
||||
public String getMail() {
|
||||
return mail;
|
||||
}
|
||||
|
||||
public void setMail(String mail) {
|
||||
this.mail = mail;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AccountDTO{" +
|
||||
"username='" + username + '\'' +
|
||||
", password='" + password + '\'' +
|
||||
", role=" + role +
|
||||
", displayName='" + displayName + '\'' +
|
||||
", department='" + department + '\'' +
|
||||
", mail='" + mail + '\'' +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,12 @@ public class AccountDO {
|
||||
|
||||
private Integer role;
|
||||
|
||||
private String displayName;
|
||||
|
||||
private String department;
|
||||
|
||||
private String mail;
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
@@ -45,16 +51,43 @@ public class AccountDO {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
public String getDisplayName() {
|
||||
return displayName;
|
||||
}
|
||||
|
||||
public void setDisplayName(String displayName) {
|
||||
this.displayName = displayName;
|
||||
}
|
||||
|
||||
public String getDepartment() {
|
||||
return department;
|
||||
}
|
||||
|
||||
public void setDepartment(String department) {
|
||||
this.department = department;
|
||||
}
|
||||
|
||||
public String getMail() {
|
||||
return mail;
|
||||
}
|
||||
|
||||
public void setMail(String mail) {
|
||||
this.mail = mail;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AccountDO{" +
|
||||
"username='" + username + '\'' +
|
||||
", password='" + password + '\'' +
|
||||
", role=" + role +
|
||||
", id=" + id +
|
||||
"id=" + id +
|
||||
", status=" + status +
|
||||
", gmtCreate=" + gmtCreate +
|
||||
", gmtModify=" + gmtModify +
|
||||
", username='" + username + '\'' +
|
||||
", password='" + password + '\'' +
|
||||
", role=" + role +
|
||||
", displayName='" + displayName + '\'' +
|
||||
", department='" + department + '\'' +
|
||||
", mail='" + mail + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.xiaojukeji.kafka.manager.common.events.metrics;
|
||||
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 22/01/17
|
||||
*/
|
||||
public class BaseMetricsCollectedEvent extends ApplicationEvent {
|
||||
/**
|
||||
* 物理集群ID
|
||||
*/
|
||||
protected final Long physicalClusterId;
|
||||
|
||||
/**
|
||||
* 收集时间,依据业务需要来设置,可以设置任务开始时间,也可以设置任务结束时间
|
||||
*/
|
||||
protected final Long collectTime;
|
||||
|
||||
public BaseMetricsCollectedEvent(Object source, Long physicalClusterId, Long collectTime) {
|
||||
super(source);
|
||||
this.physicalClusterId = physicalClusterId;
|
||||
this.collectTime = collectTime;
|
||||
}
|
||||
|
||||
public Long getPhysicalClusterId() {
|
||||
return physicalClusterId;
|
||||
}
|
||||
|
||||
public Long getCollectTime() {
|
||||
return collectTime;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.xiaojukeji.kafka.manager.common.events.metrics;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BrokerMetrics;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 20/8/31
|
||||
*/
|
||||
public class BatchBrokerMetricsCollectedEvent extends BaseMetricsCollectedEvent {
|
||||
private final List<BrokerMetrics> metricsList;
|
||||
|
||||
public BatchBrokerMetricsCollectedEvent(Object source, Long physicalClusterId, Long collectTime, List<BrokerMetrics> metricsList) {
|
||||
super(source, physicalClusterId, collectTime);
|
||||
this.metricsList = metricsList;
|
||||
}
|
||||
|
||||
public List<BrokerMetrics> getMetricsList() {
|
||||
return metricsList;
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,18 @@
|
||||
package com.xiaojukeji.kafka.manager.common.utils;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
public class BackoffUtils {
|
||||
private BackoffUtils() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 需要进行回退的事件信息
|
||||
* <回退事件名,回退结束时间>
|
||||
*/
|
||||
private static final Map<String, Long> NEED_BACK_OFF_EVENT_MAP = new ConcurrentHashMap<>();
|
||||
|
||||
public static void backoff(long timeUnitMs) {
|
||||
if (timeUnitMs <= 0) {
|
||||
return;
|
||||
@@ -17,4 +26,50 @@ public class BackoffUtils {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录回退设置
|
||||
* @param backoffEventKey 回退事件key
|
||||
* @param backoffTimeUnitMs 回退时间(ms)
|
||||
*/
|
||||
public static void putNeedBackoffEvent(String backoffEventKey, Long backoffTimeUnitMs) {
|
||||
if (backoffEventKey == null || backoffTimeUnitMs == null || backoffTimeUnitMs <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
NEED_BACK_OFF_EVENT_MAP.put(backoffEventKey, backoffTimeUnitMs + System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除回退设置
|
||||
* @param backoffEventKey 回退事件key
|
||||
*/
|
||||
public static void removeNeedBackoffEvent(String backoffEventKey) {
|
||||
NEED_BACK_OFF_EVENT_MAP.remove(backoffEventKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查是否需要回退
|
||||
* @param backoffEventKey 回退事件key
|
||||
* @return
|
||||
*/
|
||||
public static boolean isNeedBackoff(String backoffEventKey) {
|
||||
Long backoffEventEndTimeUnitMs = NEED_BACK_OFF_EVENT_MAP.get(backoffEventKey);
|
||||
if (backoffEventEndTimeUnitMs == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (backoffEventEndTimeUnitMs > System.currentTimeMillis()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 移除
|
||||
try {
|
||||
NEED_BACK_OFF_EVENT_MAP.remove(backoffEventKey, backoffEventEndTimeUnitMs);
|
||||
} catch (Exception e) {
|
||||
// 如果key不存在,这里可能出现NPE,不过不管什么异常都可以忽略
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.xiaojukeji.kafka.manager.common.utils;
|
||||
|
||||
/**
|
||||
* @className: SplitUtils
|
||||
* @description: Split string of type keyValue
|
||||
* @author: Hu.Yue
|
||||
* @date: 2021/8/4
|
||||
**/
|
||||
public class SplitUtils {
|
||||
|
||||
public static String keyValueSplit(String keyValue){
|
||||
return keyValue.split(":\\s+")[1];
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
package com.xiaojukeji.kafka.manager.common.utils.jmx;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class JmxConfig {
|
||||
/**
|
||||
* 单台最大连接数
|
||||
@@ -21,45 +26,8 @@ public class JmxConfig {
|
||||
*/
|
||||
private Boolean openSSL;
|
||||
|
||||
public Integer getMaxConn() {
|
||||
return maxConn;
|
||||
}
|
||||
|
||||
public void setMaxConn(Integer maxConn) {
|
||||
this.maxConn = maxConn;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public Boolean isOpenSSL() {
|
||||
return openSSL;
|
||||
}
|
||||
|
||||
public void setOpenSSL(Boolean openSSL) {
|
||||
this.openSSL = openSSL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "JmxConfig{" +
|
||||
"maxConn=" + maxConn +
|
||||
", username='" + username + '\'' +
|
||||
", password='" + password + '\'' +
|
||||
", openSSL=" + openSSL +
|
||||
'}';
|
||||
}
|
||||
/**
|
||||
* 连接重试回退事件
|
||||
*/
|
||||
private Long retryConnectBackoffTimeUnitMs;
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* JMXConnector包装类
|
||||
@@ -41,6 +42,8 @@ public class JmxConnectorWrap {
|
||||
|
||||
private JmxConfig jmxConfig;
|
||||
|
||||
private final ReentrantLock modifyJMXConnectorLock = new ReentrantLock();
|
||||
|
||||
public JmxConnectorWrap(Long physicalClusterId, Integer brokerId, String host, int port, JmxConfig jmxConfig) {
|
||||
this.physicalClusterId = physicalClusterId;
|
||||
this.brokerId = brokerId;
|
||||
@@ -51,7 +54,12 @@ public class JmxConnectorWrap {
|
||||
this.jmxConfig = new JmxConfig();
|
||||
}
|
||||
if (ValidateUtils.isNullOrLessThanZero(this.jmxConfig.getMaxConn())) {
|
||||
this.jmxConfig.setMaxConn(1);
|
||||
// 默认设置20
|
||||
this.jmxConfig.setMaxConn(20);
|
||||
}
|
||||
if (ValidateUtils.isNullOrLessThanZero(this.jmxConfig.getRetryConnectBackoffTimeUnitMs())) {
|
||||
// 默认回退10分钟
|
||||
this.jmxConfig.setRetryConnectBackoffTimeUnitMs(10 * 60 * 1000L);
|
||||
}
|
||||
this.atomicInteger = new AtomicInteger(this.jmxConfig.getMaxConn());
|
||||
}
|
||||
@@ -63,17 +71,40 @@ public class JmxConnectorWrap {
|
||||
if (port == -1) {
|
||||
return false;
|
||||
}
|
||||
return createJmxConnector();
|
||||
return safeCreateJmxConnector();
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
public void close() {
|
||||
this.closeJmxConnect();
|
||||
}
|
||||
|
||||
public void closeJmxConnect() {
|
||||
if (jmxConnector == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
modifyJMXConnectorLock.lock();
|
||||
|
||||
// 移除设置的backoff事件
|
||||
BackoffUtils.removeNeedBackoffEvent(buildConnectJmxFailedBackoffEventKey(physicalClusterId, brokerId));
|
||||
|
||||
jmxConnector.close();
|
||||
} catch (IOException e) {
|
||||
LOGGER.warn("close JmxConnector exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("close JmxConnector exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
|
||||
} finally {
|
||||
jmxConnector = null;
|
||||
|
||||
modifyJMXConnectorLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean safeCreateJmxConnector() {
|
||||
try {
|
||||
modifyJMXConnectorLock.lock();
|
||||
return createJmxConnector();
|
||||
} finally {
|
||||
modifyJMXConnectorLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,6 +112,12 @@ public class JmxConnectorWrap {
|
||||
if (jmxConnector != null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (BackoffUtils.isNeedBackoff(buildConnectJmxFailedBackoffEventKey(physicalClusterId, brokerId))) {
|
||||
// 被设置了需要进行回退,则本次不进行创建
|
||||
return false;
|
||||
}
|
||||
|
||||
String jmxUrl = String.format("service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", host, port);
|
||||
try {
|
||||
Map<String, Object> environment = new HashMap<String, Object>();
|
||||
@@ -88,7 +125,9 @@ public class JmxConnectorWrap {
|
||||
// fixed by riyuetianmu
|
||||
environment.put(JMXConnector.CREDENTIALS, new String[]{this.jmxConfig.getUsername(), this.jmxConfig.getPassword()});
|
||||
}
|
||||
if (jmxConfig.isOpenSSL() != null && this.jmxConfig.isOpenSSL()) {
|
||||
|
||||
if (jmxConfig.getOpenSSL() != null && this.jmxConfig.getOpenSSL()) {
|
||||
// 开启ssl
|
||||
environment.put(Context.SECURITY_PROTOCOL, "ssl");
|
||||
SslRMIClientSocketFactory clientSocketFactory = new SslRMIClientSocketFactory();
|
||||
environment.put(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, clientSocketFactory);
|
||||
@@ -96,13 +135,17 @@ public class JmxConnectorWrap {
|
||||
}
|
||||
|
||||
jmxConnector = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), environment);
|
||||
LOGGER.info("JMX connect success, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port);
|
||||
LOGGER.info("connect JMX success, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port);
|
||||
return true;
|
||||
} catch (MalformedURLException e) {
|
||||
LOGGER.error("JMX url exception, physicalClusterId:{} brokerId:{} host:{} port:{} jmxUrl:{}", physicalClusterId, brokerId, host, port, jmxUrl, e);
|
||||
LOGGER.error("connect JMX failed, JMX url exception, physicalClusterId:{} brokerId:{} host:{} port:{} jmxUrl:{}.", physicalClusterId, brokerId, host, port, jmxUrl, e);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("JMX connect exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
|
||||
LOGGER.error("connect JMX failed, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
|
||||
}
|
||||
|
||||
// 设置连接backoff
|
||||
BackoffUtils.putNeedBackoffEvent(buildConnectJmxFailedBackoffEventKey(physicalClusterId, brokerId), this.jmxConfig.getRetryConnectBackoffTimeUnitMs());
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -116,6 +159,11 @@ public class JmxConnectorWrap {
|
||||
acquire();
|
||||
MBeanServerConnection mBeanServerConnection = jmxConnector.getMBeanServerConnection();
|
||||
return mBeanServerConnection.getAttribute(name, attribute);
|
||||
} catch (IOException ioe) {
|
||||
// io错误,则重置连接
|
||||
this.closeJmxConnect();
|
||||
|
||||
throw ioe;
|
||||
} finally {
|
||||
atomicInteger.incrementAndGet();
|
||||
}
|
||||
@@ -131,6 +179,11 @@ public class JmxConnectorWrap {
|
||||
acquire();
|
||||
MBeanServerConnection mBeanServerConnection = jmxConnector.getMBeanServerConnection();
|
||||
return mBeanServerConnection.getAttributes(name, attributes);
|
||||
} catch (IOException ioe) {
|
||||
// io错误,则重置连接
|
||||
this.closeJmxConnect();
|
||||
|
||||
throw ioe;
|
||||
} finally {
|
||||
atomicInteger.incrementAndGet();
|
||||
}
|
||||
@@ -143,6 +196,11 @@ public class JmxConnectorWrap {
|
||||
acquire();
|
||||
MBeanServerConnection mBeanServerConnection = jmxConnector.getMBeanServerConnection();
|
||||
return mBeanServerConnection.queryNames(name, query);
|
||||
} catch (IOException ioe) {
|
||||
// io错误,则重置连接
|
||||
this.closeJmxConnect();
|
||||
|
||||
throw ioe;
|
||||
} finally {
|
||||
atomicInteger.incrementAndGet();
|
||||
}
|
||||
@@ -165,4 +223,8 @@ public class JmxConnectorWrap {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String buildConnectJmxFailedBackoffEventKey(Long physicalClusterId, Integer brokerId) {
|
||||
return "CONNECT_JMX_FAILED_BACK_OFF_EVENT_PHY_" + physicalClusterId + "_BROKER_" + brokerId;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,7 +295,7 @@ export class ClusterList extends SearchAndFilterContainer {
|
||||
cancelText="取消"
|
||||
okText="确认"
|
||||
>
|
||||
<Tooltip placement="left" title="暂停监控将无法正常监控指标信息,建议开启监控">
|
||||
<Tooltip placement="bottom" title="暂停监控将无法正常监控指标信息,建议开启监控">
|
||||
<a
|
||||
className="action-button"
|
||||
>
|
||||
|
||||
@@ -145,7 +145,7 @@ export const Header = observer((props: IHeader) => {
|
||||
<div className="left-content">
|
||||
<img className="kafka-header-icon" src={logoUrl} alt="" />
|
||||
<span className="kafka-header-text">LogiKM</span>
|
||||
<a className='kafka-header-version' href="https://github.com/didi/Logi-KafkaManager/releases" target='_blank'>v2.5.0</a>
|
||||
<a className='kafka-header-version' href="https://github.com/didi/Logi-KafkaManager/releases" target='_blank'>v2.6.0</a>
|
||||
{/* 添加版本超链接 */}
|
||||
</div>
|
||||
<div className="mid-content">
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -38,12 +37,10 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-web</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- javax -->
|
||||
|
||||
@@ -18,6 +18,9 @@ public class ConfigUtils {
|
||||
@Value(value = "${custom.idc:cn}")
|
||||
private String idc;
|
||||
|
||||
@Value(value = "${spring.profiles.active}")
|
||||
@Value(value = "${spring.profiles.active:dev}")
|
||||
private String kafkaManagerEnv;
|
||||
|
||||
@Value(value = "${spring.application.version:unknown}")
|
||||
private String applicationVersion;
|
||||
}
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
<result property="username" column="username" />
|
||||
<result property="password" column="password" />
|
||||
<result property="role" column="role" />
|
||||
<result property="displayName" column="display_name" />
|
||||
<result property="department" column="department" />
|
||||
<result property="mail" column="mail" />
|
||||
<result property="gmtCreate" column="gmt_create" />
|
||||
<result property="gmtModify" column="gmt_modify" />
|
||||
</resultMap>
|
||||
@@ -15,9 +18,9 @@
|
||||
<insert id="insert" parameterType="com.xiaojukeji.kafka.manager.common.entity.pojo.AccountDO">
|
||||
<![CDATA[
|
||||
REPLACE account
|
||||
(username, password, role)
|
||||
(username, password, role, display_name, department, mail)
|
||||
VALUES
|
||||
(#{username}, #{password}, #{role})
|
||||
(#{username}, #{password}, #{role}, #{displayName}, #{department}, #{mail})
|
||||
]]>
|
||||
</insert>
|
||||
|
||||
|
||||
@@ -54,7 +54,8 @@ public class BaseEnterpriseStaffService extends AbstractEnterpriseStaffService {
|
||||
}
|
||||
List<EnterpriseStaff> staffList = new ArrayList<>();
|
||||
for (AccountDO accountDO: doList) {
|
||||
staffList.add(new EnterpriseStaff(accountDO.getUsername(), accountDO.getUsername(), ""));
|
||||
//这里对chineseName填充共识的displayName,Department则获取Department信息
|
||||
staffList.add(new EnterpriseStaff(accountDO.getUsername(), accountDO.getDisplayName(), accountDO.getDepartment()));
|
||||
}
|
||||
return staffList;
|
||||
} catch (Exception e) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.account.component.ldap;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.utils.SplitUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -14,7 +15,9 @@ import javax.naming.directory.SearchControls;
|
||||
import javax.naming.directory.SearchResult;
|
||||
import javax.naming.ldap.InitialLdapContext;
|
||||
import javax.naming.ldap.LdapContext;
|
||||
import java.util.HashMap;
|
||||
import java.util.Hashtable;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
public class LdapAuthentication {
|
||||
@@ -60,8 +63,11 @@ public class LdapAuthentication {
|
||||
return null;
|
||||
}
|
||||
|
||||
private String getUserDN(String account, LdapContext ctx) {
|
||||
private Map<String, Object> getLdapAttrsInfo(String account, LdapContext ctx) {
|
||||
//存储更多的LDAP元信息
|
||||
Map<String, Object> ldapAttrsInfo = new HashMap<>();
|
||||
String userDN = "";
|
||||
ldapAttrsInfo.clear();
|
||||
try {
|
||||
SearchControls constraints = new SearchControls();
|
||||
constraints.setSearchScope(SearchControls.SUBTREE_SCOPE);
|
||||
@@ -69,7 +75,7 @@ public class LdapAuthentication {
|
||||
|
||||
NamingEnumeration<SearchResult> en = ctx.search("", filter, constraints);
|
||||
if (en == null || !en.hasMoreElements()) {
|
||||
return "";
|
||||
return null;
|
||||
}
|
||||
// maybe more than one element
|
||||
while (en.hasMoreElements()) {
|
||||
@@ -78,13 +84,25 @@ public class LdapAuthentication {
|
||||
SearchResult si = (SearchResult) obj;
|
||||
userDN += si.getName();
|
||||
userDN += "," + ldapBasedn;
|
||||
//携带LDAP更多元信息以填充用户元信息
|
||||
ldapAttrsInfo.put("userDN", userDN);
|
||||
ldapAttrsInfo.put("sAMAccountName",
|
||||
SplitUtils.keyValueSplit(si.getAttributes().get("samaccountname").toString()));
|
||||
ldapAttrsInfo.put("department",
|
||||
SplitUtils.keyValueSplit(si.getAttributes().get("department").toString()));
|
||||
ldapAttrsInfo.put("company",
|
||||
SplitUtils.keyValueSplit(si.getAttributes().get("company").toString()));
|
||||
ldapAttrsInfo.put("displayName",
|
||||
SplitUtils.keyValueSplit(si.getAttributes().get("displayname").toString()));
|
||||
ldapAttrsInfo.put("mail",
|
||||
SplitUtils.keyValueSplit(si.getAttributes().get("mail").toString()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("class=LdapAuthentication||method=getUserDN||account={}||errMsg={}", account, e);
|
||||
}
|
||||
return userDN;
|
||||
return ldapAttrsInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -93,23 +111,23 @@ public class LdapAuthentication {
|
||||
* @param password
|
||||
* @return
|
||||
*/
|
||||
public boolean authenticate(String account, String password) {
|
||||
public Map<String, Object> authenticate(String account, String password) {
|
||||
LdapContext ctx = getLdapContext();
|
||||
if (ValidateUtils.isNull(ctx)) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
String userDN = getUserDN(account, ctx);
|
||||
if(ValidateUtils.isBlank(userDN)){
|
||||
return false;
|
||||
Map<String, Object> ldapAttrsInfo = getLdapAttrsInfo(account, ctx);
|
||||
if(ValidateUtils.isNull(ldapAttrsInfo)){
|
||||
return null;
|
||||
}
|
||||
|
||||
ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN);
|
||||
ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapAttrsInfo.get("userDN").toString());
|
||||
ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password);
|
||||
ctx.reconnect(null);
|
||||
|
||||
return true;
|
||||
return ldapAttrsInfo;
|
||||
} catch (AuthenticationException e) {
|
||||
LOGGER.warn("class=LdapAuthentication||method=authenticate||account={}||errMsg={}", account, e);
|
||||
} catch (NamingException e) {
|
||||
@@ -125,6 +143,6 @@ public class LdapAuthentication {
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
@@ -47,27 +48,37 @@ public class BaseSessionSignOn extends AbstractSingleSignOn {
|
||||
if (ValidateUtils.isBlank(dto.getUsername()) || ValidateUtils.isNull(dto.getPassword())) {
|
||||
return Result.buildFailure("Missing parameters");
|
||||
}
|
||||
|
||||
Result<AccountDO> accountResult = accountService.getAccountDO(dto.getUsername());
|
||||
//先创建空对象,看是在LDAP去做填充,还是直接查表填充
|
||||
Result<AccountDO> accountResult;
|
||||
|
||||
//判断是否激活了LDAP验证, 若激活则也可使用ldap进行认证
|
||||
if(!ValidateUtils.isNull(accountLdapEnabled) && accountLdapEnabled){
|
||||
//去LDAP验证账密
|
||||
if(!ldapAuthentication.authenticate(dto.getUsername(),dto.getPassword())){
|
||||
Map<String, Object> ldapAttrsInfo = ldapAuthentication.authenticate(dto.getUsername(),dto.getPassword());;
|
||||
if(ValidateUtils.isNull(ldapAttrsInfo)){
|
||||
return Result.buildFrom(ResultStatus.LDAP_AUTHENTICATION_FAILED);
|
||||
}
|
||||
|
||||
//LDAP验证通过,拿LDAP的sAMAccountName替换dto对象的值,便于第一次自动注册采用LDAP值,并且第二次也避免REPLACE
|
||||
dto.setUsername(ldapAttrsInfo.get("sAMAccountName").toString());
|
||||
accountResult = accountService.getAccountDO(dto.getUsername());
|
||||
|
||||
if((ValidateUtils.isNull(accountResult) || ValidateUtils.isNull(accountResult.getData())) && authUserRegistration){
|
||||
//自动注册
|
||||
AccountDO accountDO = new AccountDO();
|
||||
accountDO.setUsername(dto.getUsername());
|
||||
accountDO.setRole(AccountRoleEnum.getUserRoleEnum(authUserRegistrationRole).getRole());
|
||||
accountDO.setPassword(dto.getPassword());
|
||||
accountDO.setDisplayName(ldapAttrsInfo.getOrDefault("displayName", "").toString());
|
||||
accountDO.setDepartment(ldapAttrsInfo.getOrDefault("department", "").toString());
|
||||
accountDO.setMail(ldapAttrsInfo.getOrDefault("mail", "").toString());
|
||||
accountService.createAccount(accountDO);
|
||||
}
|
||||
|
||||
return Result.buildSuc(dto.getUsername());
|
||||
}
|
||||
//不走LDAP认证直接查表填充
|
||||
accountResult = accountService.getAccountDO(dto.getUsername());
|
||||
|
||||
if (ValidateUtils.isNull(accountResult) || accountResult.failed()) {
|
||||
return new Result<>(accountResult.getCode(), accountResult.getMessage());
|
||||
|
||||
@@ -14,6 +14,7 @@ import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.servlet.http.Cookie;
|
||||
@@ -27,7 +28,13 @@ import javax.servlet.http.HttpSession;
|
||||
*/
|
||||
@Service("loginService")
|
||||
public class LoginServiceImpl implements LoginService {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LoginServiceImpl.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LoginServiceImpl.class);
|
||||
|
||||
@Value(value = "${account.jump-login.gateway-api:false}")
|
||||
private Boolean jumpLoginGatewayApi;
|
||||
|
||||
@Value(value = "${account.jump-login.third-part-api:false}")
|
||||
private Boolean jumpLoginThirdPartApi;
|
||||
|
||||
@Autowired
|
||||
private AccountService accountService;
|
||||
@@ -75,8 +82,10 @@ public class LoginServiceImpl implements LoginService {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (classRequestMappingValue.equals(ApiPrefix.API_V1_SSO_PREFIX)) {
|
||||
// 白名单接口直接true
|
||||
if (classRequestMappingValue.equals(ApiPrefix.API_V1_SSO_PREFIX) ||
|
||||
(jumpLoginGatewayApi != null && jumpLoginGatewayApi && classRequestMappingValue.equals(ApiPrefix.GATEWAY_API_V1_PREFIX)) ||
|
||||
(jumpLoginThirdPartApi != null && jumpLoginThirdPartApi && classRequestMappingValue.equals(ApiPrefix.API_V1_THIRD_PART_PREFIX))) {
|
||||
// 登录接口 or 允许跳过且是跳过类型的接口,则直接跳过登录
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,6 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -56,17 +55,14 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-beans</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -37,21 +37,24 @@ import java.util.Map;
|
||||
public class N9e extends AbstractAgent {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(N9e.class);
|
||||
|
||||
@Value("${kcm.n9e.base-url}")
|
||||
@Value("${kcm.n9e.base-url:}")
|
||||
private String baseUrl;
|
||||
|
||||
@Value("${kcm.n9e.user-token}")
|
||||
@Value("${kcm.n9e.user-token:12345678}")
|
||||
private String userToken;
|
||||
|
||||
@Value("${kcm.n9e.account}")
|
||||
@Value("${kcm.n9e.account:root}")
|
||||
private String account;
|
||||
|
||||
@Value("${kcm.n9e.timeout}")
|
||||
@Value("${kcm.n9e.timeout:300}")
|
||||
private Integer timeout;
|
||||
|
||||
@Value("${kcm.n9e.script-file}")
|
||||
@Value("${kcm.n9e.script-file:kcm_script.sh}")
|
||||
private String scriptFile;
|
||||
|
||||
@Value("${kcm.n9e.logikm-url:}")
|
||||
private String logiKMUrl;
|
||||
|
||||
private String script;
|
||||
|
||||
private static final String CREATE_TASK_URI = "/api/job-ce/tasks";
|
||||
@@ -219,7 +222,8 @@ public class N9e extends AbstractAgent {
|
||||
sb.append(creationTaskData.getKafkaPackageUrl()).append(",,");
|
||||
sb.append(creationTaskData.getServerPropertiesName().replace(KafkaFileEnum.SERVER_CONFIG.getSuffix(), "")).append(",,");
|
||||
sb.append(creationTaskData.getServerPropertiesMd5()).append(",,");
|
||||
sb.append(creationTaskData.getServerPropertiesUrl());
|
||||
sb.append(creationTaskData.getServerPropertiesUrl()).append(",,");
|
||||
sb.append(this.logiKMUrl);
|
||||
|
||||
N9eCreationTask n9eCreationTask = new N9eCreationTask();
|
||||
n9eCreationTask.setTitle(Constant.TASK_TITLE_PREFIX + "-集群ID:" + creationTaskData.getClusterId());
|
||||
|
||||
@@ -18,12 +18,13 @@ p_kafka_server_properties_name=${7} #server配置名
|
||||
p_kafka_server_properties_md5=${8} #server配置MD5
|
||||
p_kafka_server_properties_url=${9} #server配置文件下载地址
|
||||
|
||||
p_kafka_manager_url=${10} #LogiKM地址
|
||||
|
||||
#----------------------------------------配置信息------------------------------------------------------#
|
||||
g_base_dir='/home'
|
||||
g_cluster_task_dir=${g_base_dir}"/kafka_cluster_task/task_${p_task_id}" #部署升级路径
|
||||
g_rollback_version=${g_cluster_task_dir}"/rollback_version" #回滚版本
|
||||
g_new_kafka_package_name='' #最终的包名
|
||||
g_kafka_manager_addr='' #kafka-manager地址
|
||||
g_local_ip=`ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"`
|
||||
g_hostname=${g_local_ip}
|
||||
|
||||
@@ -47,7 +48,7 @@ function dchat_alarm() {
|
||||
|
||||
# 检查并初始化环境
|
||||
function check_and_init_env() {
|
||||
if [ -z "${p_task_id}" -o -z "${p_cluster_task_type}" -o -z "${p_kafka_package_url}" -o -z "${p_cluster_id}" -o -z "${p_kafka_package_name}" -o -z "${p_kafka_package_md5}" -o -z "${p_kafka_server_properties_name}" -o -z "${p_kafka_server_properties_md5}" ]; then
|
||||
if [ -z "${p_task_id}" -o -z "${p_cluster_task_type}" -o -z "${p_kafka_package_url}" -o -z "${p_cluster_id}" -o -z "${p_kafka_package_name}" -o -z "${p_kafka_package_md5}" -o -z "${p_kafka_server_properties_name}" -o -z "${p_kafka_server_properties_md5}" -o -z "${p_kafka_manager_url}" ]; then
|
||||
ECHO_LOG "存在为空的参数不合法, 退出集群任务"
|
||||
dchat_alarm "存在为空的参数不合法, 退出集群任务"
|
||||
exit 1
|
||||
@@ -72,11 +73,11 @@ function check_and_init_env() {
|
||||
|
||||
# 检查并等待集群所有的副本处于同步的状态
|
||||
function check_and_wait_broker_stabled() {
|
||||
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${g_kafka_manager_addr}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
|
||||
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${p_kafka_manager_url}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
|
||||
while [ "$under_replication_count" -ne 1 ]; do
|
||||
ECHO_LOG "存在${under_replication_count}个副本未同步, sleep 10s"
|
||||
sleep 10
|
||||
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${g_kafka_manager_addr}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
|
||||
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${p_kafka_manager_url}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
|
||||
done
|
||||
ECHO_LOG "集群副本都已经处于同步的状态, 可以进行集群升级"
|
||||
}
|
||||
@@ -324,6 +325,7 @@ ECHO_LOG " p_kafka_package_name=${p_kafka_package_name}"
|
||||
ECHO_LOG " p_kafka_package_md5=${p_kafka_package_md5}"
|
||||
ECHO_LOG " p_kafka_server_properties_name=${p_kafka_server_properties_name}"
|
||||
ECHO_LOG " p_kafka_server_properties_md5=${p_kafka_server_properties_md5}"
|
||||
ECHO_LOG " p_kafka_manager_url=${p_kafka_manager_url}"
|
||||
|
||||
|
||||
|
||||
@@ -342,7 +344,7 @@ fi
|
||||
ECHO_LOG "停kafka服务"
|
||||
stop_kafka_server
|
||||
|
||||
ECHO_LOG "停5秒, 确保"
|
||||
ECHO_LOG "再停5秒, 确保端口已释放"
|
||||
sleep 5
|
||||
|
||||
if [ "${p_cluster_task_type}" == "0" ];then
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -63,12 +62,10 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-beans</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -25,7 +25,6 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -48,7 +47,6 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -24,7 +24,6 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<description>
|
||||
@@ -46,7 +45,6 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -24,7 +24,6 @@
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<file_encoding>UTF-8</file_encoding>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -52,7 +51,6 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.component;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.factory.DefaultThreadFactory;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.JsonUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.NetUtils;
|
||||
@@ -29,7 +28,7 @@ import java.util.concurrent.*;
|
||||
* @date 20/8/10
|
||||
*/
|
||||
public abstract class AbstractScheduledTask<E extends Comparable> implements SchedulingConfigurer {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractScheduledTask.class);
|
||||
|
||||
@Autowired
|
||||
private HeartbeatDao heartbeatDao;
|
||||
@@ -73,18 +72,16 @@ public abstract class AbstractScheduledTask<E extends Comparable> implements Sch
|
||||
LOGGER.info("init custom scheduled finished, scheduledName:{} scheduledCron:{}.", scheduledName, scheduledCron);
|
||||
}
|
||||
|
||||
private boolean checkAndModifyCron(String scheduledName, String scheduledCron, boolean existIfIllegal) {
|
||||
private boolean checkAndModifyCron(String scheduledName, String scheduledCron, boolean isInit) {
|
||||
if (scheduledCron.matches(ScheduledTaskConstant.CRON_REG_EX)) {
|
||||
this.scheduledCron = scheduledCron;
|
||||
LOGGER.info("modify scheduledCron success, scheduledName:{} scheduledCron:{}."
|
||||
, scheduledName, scheduledCron);
|
||||
LOGGER.info("{} scheduledCron success, scheduledName:{} scheduledCron:{}.", isInit? "init": "modify", scheduledName, scheduledCron);
|
||||
return true;
|
||||
}
|
||||
|
||||
LOGGER.error("modify scheduledCron failed, format invalid, scheduledName:{} scheduledCron:{}."
|
||||
, scheduledName, scheduledCron);
|
||||
if (existIfIllegal) {
|
||||
System.exit(0);
|
||||
LOGGER.error("modify scheduledCron failed, format invalid, scheduledName:{} scheduledCron:{}.", scheduledName, scheduledCron);
|
||||
if (isInit) {
|
||||
throw new UnsupportedOperationException(String.format("scheduledName:%s scheduledCron:%s format invalid", scheduledName, scheduledCron));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -130,7 +127,8 @@ public abstract class AbstractScheduledTask<E extends Comparable> implements Sch
|
||||
LOGGER.info("customScheduled task finished, empty selected task, scheduledName:{}.", scheduledName);
|
||||
return;
|
||||
}
|
||||
LOGGER.info("customScheduled task running, selected tasks, IP:{} selectedTasks:{}.",
|
||||
|
||||
LOGGER.debug("customScheduled task running, selected tasks, IP:{} selectedTasks:{}.",
|
||||
NetUtils.localIp(), JsonUtils.toJSONString(selectTasks)
|
||||
);
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.component;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@@ -9,11 +8,11 @@ import org.slf4j.LoggerFactory;
|
||||
* @date 20/8/10
|
||||
*/
|
||||
public class BaseBizTask<E extends Comparable> implements Runnable {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractScheduledTask.class);
|
||||
|
||||
private E task;
|
||||
private final E task;
|
||||
|
||||
private AbstractScheduledTask scheduledTask;
|
||||
private final AbstractScheduledTask scheduledTask;
|
||||
|
||||
public BaseBizTask(E task, AbstractScheduledTask scheduledTask) {
|
||||
this.task = task;
|
||||
@@ -30,6 +29,7 @@ public class BaseBizTask<E extends Comparable> implements Runnable {
|
||||
} catch (Throwable t) {
|
||||
LOGGER.error("scheduled task scheduleName:{} execute failed, task:{}", scheduledTask.getScheduledName(), task, t);
|
||||
}
|
||||
|
||||
LOGGER.info("scheduled task scheduleName:{} finished, cost-time:{}ms.", scheduledTask.getScheduledName(), System.currentTimeMillis() - startTime);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.component;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.NetUtils;
|
||||
import com.xiaojukeji.kafka.manager.dao.HeartbeatDao;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.HeartbeatDO;
|
||||
@@ -18,7 +17,7 @@ import java.util.Date;
|
||||
*/
|
||||
@Component
|
||||
public class Heartbeat {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(Heartbeat.class);
|
||||
|
||||
@Autowired
|
||||
private HeartbeatDao heartbeatDao;
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.biz;
|
||||
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.task.config.TopicBillConfig;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.DateUtils;
|
||||
@@ -30,7 +28,7 @@ import java.util.*;
|
||||
*/
|
||||
@CustomScheduled(name = "calKafkaBill", cron = "0 0 1 * * *", threadNum = 1)
|
||||
public class CalKafkaTopicBill extends AbstractScheduledTask<ClusterDO> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(CalKafkaTopicBill.class);
|
||||
|
||||
@Autowired
|
||||
private AppService appService;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.biz;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicStatisticsDO;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.DateUtils;
|
||||
@@ -30,7 +29,7 @@ import java.util.Map;
|
||||
*/
|
||||
@CustomScheduled(name = "calTopicStatistics", cron = "0 0 0/4 * * ?", threadNum = 5)
|
||||
public class CalTopicStatistics extends AbstractScheduledTask<ClusterDO> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(CalTopicStatistics.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.biz;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.DBStatusEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.BrokerDO;
|
||||
@@ -27,7 +26,7 @@ import java.util.*;
|
||||
*/
|
||||
@CustomScheduled(name = "flushBrokerTable", cron = "0 0 0/1 * * ?", threadNum = 1)
|
||||
public class FlushBrokerTable extends AbstractScheduledTask<ClusterDO> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushBrokerTable.class);
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.biz;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicExpiredDO;
|
||||
@@ -32,7 +31,7 @@ import java.util.Map;
|
||||
*/
|
||||
@CustomScheduled(name = "flushExpiredTopic", cron = "0 0 0/5 * * ?", threadNum = 1)
|
||||
public class FlushExpiredTopic extends AbstractScheduledTask<ClusterDO> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushExpiredTopic.class);
|
||||
|
||||
@Autowired
|
||||
private TopicExpiredDao topicExpiredDao;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.biz;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterTaskDO;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.kcm.ClusterTaskService;
|
||||
@@ -24,7 +23,7 @@ import java.util.List;
|
||||
@CustomScheduled(name = "syncClusterTaskState", cron = "0 0/1 * * * ?", threadNum = 1)
|
||||
@ConditionalOnProperty(prefix = "kcm", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SyncClusterTaskState extends AbstractScheduledTask<EmptyEntry> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SyncClusterTaskState.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterTaskService clusterTaskService;
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.metrics.collect;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.events.metrics.BatchBrokerMetricsCollectedEvent;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.SpringTool;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
|
||||
import com.xiaojukeji.kafka.manager.service.service.ClusterService;
|
||||
import com.xiaojukeji.kafka.manager.service.service.JmxService;
|
||||
import com.xiaojukeji.kafka.manager.service.strategy.AbstractHealthScoreStrategy;
|
||||
import com.xiaojukeji.kafka.manager.task.component.AbstractScheduledTask;
|
||||
import com.xiaojukeji.kafka.manager.task.component.CustomScheduled;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Broker指标信息收集
|
||||
* @author zengqiao
|
||||
* @date 20/5/7
|
||||
*/
|
||||
@CustomScheduled(name = "collectAndPublishBrokerMetrics", cron = "21 0/1 * * * ?", threadNum = 2)
|
||||
@ConditionalOnProperty(prefix = "task.metrics.collect", name = "broker-metrics-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class CollectAndPublishBrokerMetrics extends AbstractScheduledTask<ClusterDO> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(CollectAndPublishBrokerMetrics.class);
|
||||
|
||||
@Autowired
|
||||
private JmxService jmxService;
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@Autowired
|
||||
private AbstractHealthScoreStrategy healthScoreStrategy;
|
||||
|
||||
@Override
|
||||
protected List<ClusterDO> listAllTasks() {
|
||||
return clusterService.list();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void processTask(ClusterDO clusterDO) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try {
|
||||
SpringTool.publish(new BatchBrokerMetricsCollectedEvent(
|
||||
this,
|
||||
clusterDO.getId(),
|
||||
startTime,
|
||||
this.getBrokerMetrics(clusterDO.getId()))
|
||||
);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("collect broker-metrics failed, physicalClusterId:{}.", clusterDO.getId(), e);
|
||||
}
|
||||
|
||||
LOGGER.info("collect broker-metrics finished, physicalClusterId:{} costTime:{}", clusterDO.getId(), System.currentTimeMillis() - startTime);
|
||||
}
|
||||
|
||||
private List<BrokerMetrics> getBrokerMetrics(Long clusterId) {
|
||||
List<BrokerMetrics> metricsList = new ArrayList<>();
|
||||
for (Integer brokerId: PhysicalClusterMetadataManager.getBrokerIdList(clusterId)) {
|
||||
BrokerMetrics metrics = jmxService.getBrokerMetrics(
|
||||
clusterId,
|
||||
brokerId,
|
||||
KafkaMetricsCollections.BROKER_TO_DB_METRICS
|
||||
);
|
||||
|
||||
if (ValidateUtils.isNull(metrics)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
metrics.getMetricsMap().put(
|
||||
JmxConstant.HEALTH_SCORE,
|
||||
healthScoreStrategy.calBrokerHealthScore(clusterId, brokerId, metrics)
|
||||
);
|
||||
|
||||
metricsList.add(metrics);
|
||||
}
|
||||
|
||||
if (ValidateUtils.isEmptyList(metricsList)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
return metricsList;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.metrics.collect;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.ConsumerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
@@ -34,7 +33,7 @@ import java.util.concurrent.FutureTask;
|
||||
*/
|
||||
@CustomScheduled(name = "newCollectAndPublishCGData", cron = "30 0/1 * * * *", threadNum = 10)
|
||||
public class CollectAndPublishCGData extends AbstractScheduledTask<ClusterDO> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(CollectAndPublishCGData.class);
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.metrics.delete;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.BackoffUtils;
|
||||
import com.xiaojukeji.kafka.manager.dao.*;
|
||||
import com.xiaojukeji.kafka.manager.task.component.AbstractScheduledTask;
|
||||
@@ -22,7 +21,7 @@ import java.util.List;
|
||||
*/
|
||||
@CustomScheduled(name = "deleteMetrics", cron = "0 0/2 * * * ?", threadNum = 1)
|
||||
public class DeleteMetrics extends AbstractScheduledTask<EmptyEntry> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(DeleteMetrics.class);
|
||||
|
||||
@Autowired
|
||||
private TopicMetricsDao topicMetricsDao;
|
||||
@@ -42,25 +41,25 @@ public class DeleteMetrics extends AbstractScheduledTask<EmptyEntry> {
|
||||
@Autowired
|
||||
private TopicThrottledMetricsDao topicThrottledMetricsDao;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.delete-limit-size:1000}")
|
||||
@Value(value = "${task.metrics.delete.delete-limit-size:1000}")
|
||||
private Integer deleteLimitSize;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.cluster-metrics-save-days:14}")
|
||||
@Value(value = "${task.metrics.delete.cluster-metrics-save-days:14}")
|
||||
private Integer clusterMetricsSaveDays;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.broker-metrics-save-days:14}")
|
||||
@Value(value = "${task.metrics.delete.broker-metrics-save-days:14}")
|
||||
private Integer brokerMetricsSaveDays;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.topic-metrics-save-days:7}")
|
||||
@Value(value = "${task.metrics.delete.topic-metrics-save-days:7}")
|
||||
private Integer topicMetricsSaveDays;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.topic-request-time-metrics-save-days:7}")
|
||||
@Value(value = "${task.metrics.delete.topic-request-time-metrics-save-days:7}")
|
||||
private Integer topicRequestTimeMetricsSaveDays;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.topic-throttled-metrics-save-days:7}")
|
||||
@Value(value = "${task.metrics.delete.topic-throttled-metrics-save-days:7}")
|
||||
private Integer topicThrottledMetricsSaveDays;
|
||||
|
||||
@Value(value = "${task.metrics.delete-metrics.app-topic-metrics-save-days:7}")
|
||||
@Value(value = "${task.metrics.delete.app-topic-metrics-save-days:7}")
|
||||
private Integer appTopicMetricsSaveDays;
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
package com.xiaojukeji.kafka.manager.task.dispatch.metrics.store;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata;
|
||||
import com.xiaojukeji.kafka.manager.dao.BrokerMetricsDao;
|
||||
import com.xiaojukeji.kafka.manager.dao.ClusterMetricsDao;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.BrokerMetricsDO;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterMetricsDO;
|
||||
import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
|
||||
import com.xiaojukeji.kafka.manager.service.service.ClusterService;
|
||||
import com.xiaojukeji.kafka.manager.service.service.JmxService;
|
||||
import com.xiaojukeji.kafka.manager.service.strategy.AbstractHealthScoreStrategy;
|
||||
import com.xiaojukeji.kafka.manager.service.utils.MetricsConvertUtils;
|
||||
import com.xiaojukeji.kafka.manager.task.component.AbstractScheduledTask;
|
||||
import com.xiaojukeji.kafka.manager.task.component.CustomScheduled;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Broker指标信息存DB, Broker流量, 集群流量
|
||||
* @author zengqiao
|
||||
* @date 20/5/7
|
||||
*/
|
||||
@CustomScheduled(name = "storeBrokerMetrics", cron = "21 0/1 * * * ?", threadNum = 2)
|
||||
@ConditionalOnProperty(prefix = "custom.store-metrics-task.community", name = "broker-metrics-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class StoreBrokerMetrics extends AbstractScheduledTask<ClusterDO> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
|
||||
@Autowired
|
||||
private JmxService jmxService;
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@Autowired
|
||||
private BrokerMetricsDao brokerMetricsDao;
|
||||
|
||||
@Autowired
|
||||
private ClusterMetricsDao clusterMetricsDao;
|
||||
|
||||
@Autowired
|
||||
private AbstractHealthScoreStrategy healthScoreStrategy;
|
||||
|
||||
@Override
|
||||
protected List<ClusterDO> listAllTasks() {
|
||||
return clusterService.list();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void processTask(ClusterDO clusterDO) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
List<ClusterMetrics> clusterMetricsList = new ArrayList<>();
|
||||
|
||||
try {
|
||||
List<BrokerMetrics> brokerMetricsList = getAndBatchAddMetrics(startTime, clusterDO.getId());
|
||||
clusterMetricsList.add(supplyAndConvert2ClusterMetrics(
|
||||
clusterDO.getId(),
|
||||
MetricsConvertUtils.merge2BaseMetricsByAdd(brokerMetricsList))
|
||||
);
|
||||
} catch (Exception t) {
|
||||
LOGGER.error("collect failed, clusterId:{}.", clusterDO.getId(), t);
|
||||
}
|
||||
long endTime = System.currentTimeMillis();
|
||||
LOGGER.info("collect finish, clusterId:{} costTime:{}", clusterDO.getId(), endTime - startTime);
|
||||
|
||||
List<ClusterMetricsDO> doList = MetricsConvertUtils.convertAndUpdateCreateTime2ClusterMetricsDOList(
|
||||
startTime,
|
||||
clusterMetricsList
|
||||
);
|
||||
|
||||
if (ValidateUtils.isEmptyList(doList)) {
|
||||
return;
|
||||
}
|
||||
|
||||
clusterMetricsDao.batchAdd(doList);
|
||||
}
|
||||
|
||||
private List<BrokerMetrics> getAndBatchAddMetrics(Long startTime, Long clusterId) {
|
||||
List<BrokerMetrics> metricsList = new ArrayList<>();
|
||||
for (Integer brokerId: PhysicalClusterMetadataManager.getBrokerIdList(clusterId)) {
|
||||
BrokerMetrics metrics = jmxService.getBrokerMetrics(
|
||||
clusterId,
|
||||
brokerId,
|
||||
KafkaMetricsCollections.BROKER_TO_DB_METRICS
|
||||
);
|
||||
if (ValidateUtils.isNull(metrics)) {
|
||||
continue;
|
||||
}
|
||||
metrics.getMetricsMap().put(
|
||||
JmxConstant.HEALTH_SCORE,
|
||||
healthScoreStrategy.calBrokerHealthScore(clusterId, brokerId, metrics)
|
||||
);
|
||||
metricsList.add(metrics);
|
||||
}
|
||||
if (ValidateUtils.isEmptyList(metricsList)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
List<BrokerMetricsDO> doList =
|
||||
MetricsConvertUtils.convertAndUpdateCreateTime2BrokerMetricsDOList(startTime, metricsList);
|
||||
int i = 0;
|
||||
do {
|
||||
List<BrokerMetricsDO> subDOList = doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size()));
|
||||
if (ValidateUtils.isEmptyList(subDOList)) {
|
||||
break;
|
||||
}
|
||||
|
||||
brokerMetricsDao.batchAdd(subDOList);
|
||||
i += Constant.BATCH_INSERT_SIZE;
|
||||
} while (i < doList.size());
|
||||
|
||||
return metricsList;
|
||||
}
|
||||
|
||||
private ClusterMetrics supplyAndConvert2ClusterMetrics(Long clusterId, BaseMetrics baseMetrics) {
|
||||
ClusterMetrics metrics = new ClusterMetrics(clusterId);
|
||||
Map<String, Object> metricsMap = metrics.getMetricsMap();
|
||||
metricsMap.putAll(baseMetrics.getMetricsMap());
|
||||
metricsMap.put(JmxConstant.TOPIC_NUM, PhysicalClusterMetadataManager.getTopicNameList(clusterId).size());
|
||||
metricsMap.put(JmxConstant.BROKER_NUM, PhysicalClusterMetadataManager.getBrokerIdList(clusterId).size());
|
||||
Integer partitionNum = 0;
|
||||
for (String topicName : PhysicalClusterMetadataManager.getTopicNameList(clusterId)) {
|
||||
TopicMetadata topicMetaData = PhysicalClusterMetadataManager.getTopicMetadata(clusterId, topicName);
|
||||
if (ValidateUtils.isNull(topicMetaData)) {
|
||||
continue;
|
||||
}
|
||||
partitionNum += topicMetaData.getPartitionNum();
|
||||
}
|
||||
metricsMap.put(JmxConstant.PARTITION_NUM, partitionNum);
|
||||
return metrics;
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package com.xiaojukeji.kafka.manager.task.dispatch.metrics.store;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.dao.TopicAppMetricsDao;
|
||||
@@ -28,7 +27,7 @@ import java.util.*;
|
||||
@CustomScheduled(name = "storeDiDiAppTopicMetrics", cron = "41 0/1 * * * ?", threadNum = 5)
|
||||
@ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "app-topic-metrics-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class StoreDiDiAppTopicMetrics extends AbstractScheduledTask<ClusterDO> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(StoreDiDiAppTopicMetrics.class);
|
||||
|
||||
@Autowired
|
||||
private JmxService jmxService;
|
||||
|
||||
@@ -2,7 +2,6 @@ package com.xiaojukeji.kafka.manager.task.dispatch.metrics.store;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.dao.TopicRequestMetricsDao;
|
||||
@@ -28,7 +27,7 @@ import java.util.*;
|
||||
@CustomScheduled(name = "storeDiDiTopicRequestTimeMetrics", cron = "51 0/1 * * * ?", threadNum = 5)
|
||||
@ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "topic-request-time-metrics-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class StoreDiDiTopicRequestTimeMetrics extends AbstractScheduledTask<ClusterDO> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(StoreDiDiTopicRequestTimeMetrics.class);
|
||||
|
||||
@Autowired
|
||||
private JmxService jmxService;
|
||||
|
||||
@@ -4,7 +4,6 @@ import com.xiaojukeji.kafka.manager.bpm.OrderService;
|
||||
import com.xiaojukeji.kafka.manager.bpm.common.OrderStatusEnum;
|
||||
import com.xiaojukeji.kafka.manager.bpm.common.OrderTypeEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.SystemCodeConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
|
||||
@@ -39,7 +38,7 @@ import java.util.Properties;
|
||||
@CustomScheduled(name = "autoHandleTopicOrder", cron = "0 0/1 * * * ?", threadNum = 1)
|
||||
@ConditionalOnProperty(prefix = "task.op.order-auto-exec", name = "topic-enabled", havingValue = "true", matchIfMissing = false)
|
||||
public class AutoHandleTopicOrder extends AbstractScheduledTask<EmptyEntry> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AutoHandleTopicOrder.class);
|
||||
|
||||
@Autowired
|
||||
private ConfigService configService;
|
||||
|
||||
@@ -4,7 +4,6 @@ import com.xiaojukeji.kafka.manager.bpm.OrderService;
|
||||
import com.xiaojukeji.kafka.manager.bpm.common.OrderStatusEnum;
|
||||
import com.xiaojukeji.kafka.manager.bpm.common.OrderTypeEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
|
||||
import com.xiaojukeji.kafka.manager.bpm.common.handle.OrderHandleBaseDTO;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.DateUtils;
|
||||
@@ -34,7 +33,7 @@ import java.util.*;
|
||||
@CustomScheduled(name = "automatedHandleOrder", cron = "0 0/1 * * * ?", threadNum = 1)
|
||||
@ConditionalOnProperty(prefix = "task.op.order-auto-exec", name = "app-enabled", havingValue = "true", matchIfMissing = false)
|
||||
public class AutomatedHandleOrder extends AbstractScheduledTask<EmptyEntry> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AutomatedHandleOrder.class);
|
||||
|
||||
@Autowired
|
||||
private OrderService orderService;
|
||||
|
||||
@@ -3,7 +3,6 @@ package com.xiaojukeji.kafka.manager.task.dispatch.op;
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.TaskStatusEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.TaskStatusReassignEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
@@ -36,7 +35,7 @@ import java.util.*;
|
||||
@Component
|
||||
@CustomScheduled(name = "flushReassignment", cron = "0 0/1 * * * ?", threadNum = 1)
|
||||
public class FlushReassignment extends AbstractScheduledTask<EmptyEntry> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushReassignment.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@@ -2,7 +2,6 @@ package com.xiaojukeji.kafka.manager.task.dispatch.op;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.TopicAuthorityEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO;
|
||||
@@ -39,7 +38,7 @@ import java.util.stream.Collectors;
|
||||
@CustomScheduled(name = "syncTopic2DB", cron = "0 0/2 * * * ?", threadNum = 1)
|
||||
@ConditionalOnProperty(prefix = "task.op", name = "sync-topic-enabled", havingValue = "true", matchIfMissing = false)
|
||||
public class SyncTopic2DB extends AbstractScheduledTask<EmptyEntry> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SyncTopic2DB.class);
|
||||
|
||||
private static final String SYNC_TOPIC_2_DB_CONFIG_KEY = "SYNC_TOPIC_2_DB_CONFIG_KEY";
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.db;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.BrokerMetricsDO;
|
||||
import com.xiaojukeji.kafka.manager.common.events.metrics.BatchBrokerMetricsCollectedEvent;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.dao.BrokerMetricsDao;
|
||||
import com.xiaojukeji.kafka.manager.service.utils.MetricsConvertUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 22/01/17
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "task.metrics.sink.broker-metrics", name = "sink-db-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SinkBrokerMetrics2DB implements ApplicationListener<BatchBrokerMetricsCollectedEvent> {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SinkBrokerMetrics2DB.class);
|
||||
|
||||
@Autowired
|
||||
private BrokerMetricsDao metricsDao;
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(BatchBrokerMetricsCollectedEvent event) {
|
||||
logger.debug("sink broker-metrics to db start, event:{}.", event);
|
||||
|
||||
List<BrokerMetrics> metricsList = event.getMetricsList();
|
||||
if (ValidateUtils.isEmptyList(metricsList)) {
|
||||
logger.warn("sink broker-metrics to db finished, without need sink, event:{}.", event);
|
||||
return;
|
||||
}
|
||||
|
||||
List<BrokerMetricsDO> doList = MetricsConvertUtils.convertAndUpdateCreateTime2BrokerMetricsDOList(event.getCollectTime(), metricsList);
|
||||
int i = 0;
|
||||
while (i < doList.size()) {
|
||||
List<BrokerMetricsDO> subDOList = doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size()));
|
||||
if (ValidateUtils.isEmptyList(subDOList)) {
|
||||
break;
|
||||
}
|
||||
|
||||
metricsDao.batchAdd(subDOList);
|
||||
i += Constant.BATCH_INSERT_SIZE;
|
||||
}
|
||||
|
||||
logger.debug("sink broker-metrics to db finished, event:{}.", event);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.db;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterMetricsDO;
|
||||
import com.xiaojukeji.kafka.manager.common.events.metrics.BatchBrokerMetricsCollectedEvent;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata;
|
||||
import com.xiaojukeji.kafka.manager.dao.ClusterMetricsDao;
|
||||
import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
|
||||
import com.xiaojukeji.kafka.manager.service.utils.MetricsConvertUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 22/01/17
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "task.metrics.sink.cluster-metrics", name = "sink-db-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SinkClusterMetrics2DB implements ApplicationListener<BatchBrokerMetricsCollectedEvent> {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SinkClusterMetrics2DB.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterMetricsDao clusterMetricsDao;
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(BatchBrokerMetricsCollectedEvent event) {
|
||||
logger.debug("sink cluster-metrics to db start, event:{}.", event);
|
||||
|
||||
List<BrokerMetrics> metricsList = event.getMetricsList();
|
||||
if (ValidateUtils.isEmptyList(metricsList)) {
|
||||
logger.warn("sink cluster-metrics to db finished, without need sink, event:{}.", event);
|
||||
return;
|
||||
}
|
||||
|
||||
List<ClusterMetricsDO> doList = MetricsConvertUtils.convertAndUpdateCreateTime2ClusterMetricsDOList(
|
||||
event.getCollectTime(),
|
||||
// 合并broker-metrics为cluster-metrics
|
||||
Arrays.asList(supplyAndConvert2ClusterMetrics(event.getPhysicalClusterId(), MetricsConvertUtils.merge2BaseMetricsByAdd(event.getMetricsList())))
|
||||
);
|
||||
|
||||
if (ValidateUtils.isEmptyList(doList)) {
|
||||
logger.warn("sink cluster-metrics to db finished, without need sink, event:{}.", event);
|
||||
return;
|
||||
}
|
||||
|
||||
clusterMetricsDao.batchAdd(doList);
|
||||
|
||||
logger.debug("sink cluster-metrics to db finished, event:{}.", event);
|
||||
}
|
||||
|
||||
private ClusterMetrics supplyAndConvert2ClusterMetrics(Long clusterId, BaseMetrics baseMetrics) {
|
||||
ClusterMetrics metrics = new ClusterMetrics(clusterId);
|
||||
Map<String, Object> metricsMap = metrics.getMetricsMap();
|
||||
metricsMap.putAll(baseMetrics.getMetricsMap());
|
||||
metricsMap.put(JmxConstant.TOPIC_NUM, PhysicalClusterMetadataManager.getTopicNameList(clusterId).size());
|
||||
metricsMap.put(JmxConstant.BROKER_NUM, PhysicalClusterMetadataManager.getBrokerIdList(clusterId).size());
|
||||
Integer partitionNum = 0;
|
||||
for (String topicName : PhysicalClusterMetadataManager.getTopicNameList(clusterId)) {
|
||||
TopicMetadata topicMetaData = PhysicalClusterMetadataManager.getTopicMetadata(clusterId, topicName);
|
||||
if (ValidateUtils.isNull(topicMetaData)) {
|
||||
continue;
|
||||
}
|
||||
partitionNum += topicMetaData.getPartitionNum();
|
||||
}
|
||||
metricsMap.put(JmxConstant.PARTITION_NUM, partitionNum);
|
||||
return metrics;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.db;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicMetricsDO;
|
||||
import com.xiaojukeji.kafka.manager.common.events.TopicMetricsCollectedEvent;
|
||||
@@ -25,7 +24,7 @@ import java.util.List;
|
||||
@Component("storeCommunityTopicMetrics2DB")
|
||||
@ConditionalOnProperty(prefix = "custom.store-metrics-task.community", name = "topic-metrics-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class StoreCommunityTopicMetrics2DB implements ApplicationListener<TopicMetricsCollectedEvent> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(StoreCommunityTopicMetrics2DB.class);
|
||||
|
||||
@Autowired
|
||||
private TopicMetricsDao topicMetricsDao;
|
||||
@@ -1,8 +1,7 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.db;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.KafkaClientEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.Constant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicThrottledMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicThrottledMetricsDO;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
@@ -24,7 +23,7 @@ import java.util.*;
|
||||
@Component("storeTopicThrottledMetrics2DB")
|
||||
@ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "topic-throttled-metrics-enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class StoreTopicThrottledMetrics2DB implements ApplicationListener<TopicThrottledMetricsCollectedEvent> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(StoreTopicThrottledMetrics2DB.class);
|
||||
|
||||
@Autowired
|
||||
private ThrottleService throttleService;
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.kafka;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.ConfigConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ao.config.TopicNameConfig;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ao.remote.KafkaTopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
@@ -27,7 +26,7 @@ import java.util.List;
|
||||
*/
|
||||
@Component("sinkCommunityTopicMetrics2Kafka")
|
||||
public class SinkCommunityTopicMetrics2Kafka implements ApplicationListener<TopicMetricsCollectedEvent> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SinkCommunityTopicMetrics2Kafka.class);
|
||||
|
||||
@Autowired
|
||||
private ConfigService configService;
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.kafka;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.ConfigConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ao.config.TopicNameConfig;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ao.remote.KafkaConsumerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ao.remote.KafkaConsumerMetricsElem;
|
||||
@@ -27,7 +26,7 @@ import java.util.Map;
|
||||
*/
|
||||
@Component("produceConsumerMetrics")
|
||||
public class SinkConsumerMetrics2Kafka implements ApplicationListener<ConsumerMetricsCollectedEvent> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SinkConsumerMetrics2Kafka.class);
|
||||
|
||||
@Autowired
|
||||
private ConfigService configService;
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.monitor;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.monitor.common.entry.bizenum.MonitorMetricNameEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.monitor.common.MonitorSinkConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.monitor.common.entry.sink.MonitorTopicSinkTag;
|
||||
@@ -33,7 +32,7 @@ import java.util.List;
|
||||
@ConditionalOnProperty(prefix = "monitor", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
@CustomScheduled(name = "sinkCommunityTopicMetrics2Monitor", cron = "1 0/1 * * * ?", threadNum = 5)
|
||||
public class SinkCommunityTopicMetrics2Monitor extends AbstractScheduledTask<ClusterDO> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SinkCommunityTopicMetrics2Monitor.class);
|
||||
|
||||
@Autowired
|
||||
private AbstractMonitorService abstractMonitor;
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.monitor;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.monitor.common.entry.bizenum.MonitorMetricNameEnum;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.monitor.common.MonitorSinkConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.ConsumerMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
@@ -32,7 +31,7 @@ import java.util.*;
|
||||
@Component("sinkConsumerMetrics2Monitor")
|
||||
@ConditionalOnProperty(prefix = "monitor", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public class SinkConsumerMetrics2Monitor implements ApplicationListener<ConsumerMetricsCollectedEvent> {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SinkConsumerMetrics2Monitor.class);
|
||||
|
||||
@Autowired
|
||||
private LogicalClusterMetadataManager logicalClusterMetadataManager;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.xiaojukeji.kafka.manager.task.listener;
|
||||
package com.xiaojukeji.kafka.manager.task.listener.sink.monitor;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.bizenum.KafkaClientEnum;
|
||||
import com.xiaojukeji.kafka.manager.monitor.common.MonitorSinkConstant;
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.task.schedule;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
@@ -22,7 +21,7 @@ import java.util.*;
|
||||
*/
|
||||
@Component
|
||||
public class FlushTopicMetrics {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushTopicMetrics.class);
|
||||
|
||||
@Autowired
|
||||
private JmxService jmxService;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.schedule.metadata;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ConsumerMetadata;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
|
||||
@@ -25,7 +24,7 @@ import java.util.*;
|
||||
*/
|
||||
@Component
|
||||
public class FlushBKConsumerGroupMetadata {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushBKConsumerGroupMetadata.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.schedule.metadata;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl;
|
||||
import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils;
|
||||
@@ -22,7 +21,7 @@ import java.util.Properties;
|
||||
*/
|
||||
@Component
|
||||
public class FlushTopicProperties {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushTopicProperties.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.xiaojukeji.kafka.manager.task.schedule.metadata;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.entity.ConsumerMetadata;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl;
|
||||
@@ -27,7 +26,7 @@ import java.util.stream.Collectors;
|
||||
*/
|
||||
@Component
|
||||
public class FlushZKConsumerGroupMetadata {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(FlushZKConsumerGroupMetadata.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterService clusterService;
|
||||
|
||||
@@ -16,8 +16,6 @@
|
||||
<properties>
|
||||
<java_source_version>1.8</java_source_version>
|
||||
<java_target_version>1.8</java_target_version>
|
||||
<springframework.boot.version>2.1.1.RELEASE</springframework.boot.version>
|
||||
<spring-version>5.1.3.RELEASE</spring-version>
|
||||
<failOnMissingWebXml>false</failOnMissingWebXml>
|
||||
<tomcat.version>8.5.72</tomcat.version>
|
||||
<log4j2.version>2.16.0</log4j2.version>
|
||||
@@ -73,22 +71,22 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
<version>${springframework.boot.version}</version>
|
||||
<version>${spring.boot.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-aop</artifactId>
|
||||
<version>${springframework.boot.version}</version>
|
||||
<version>${spring.boot.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
<version>${springframework.boot.version}</version>
|
||||
<version>${spring.boot.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-thymeleaf</artifactId>
|
||||
<version>${springframework.boot.version}</version>
|
||||
<version>${spring.boot.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
@@ -105,18 +103,17 @@
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context-support</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<finalName>kafka-manager</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>${springframework.boot.version}</version>
|
||||
<version>${spring.boot.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
|
||||
@@ -3,7 +3,6 @@ package com.xiaojukeji.kafka.manager.web;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.web.servlet.ServletComponentScan;
|
||||
import org.springframework.scheduling.annotation.EnableAsync;
|
||||
@@ -17,7 +16,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
@EnableAsync
|
||||
@EnableScheduling
|
||||
@ServletComponentScan
|
||||
@EnableAutoConfiguration
|
||||
@SpringBootApplication(scanBasePackages = {"com.xiaojukeji.kafka.manager"})
|
||||
public class MainApplication {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MainApplication.class);
|
||||
@@ -28,7 +26,8 @@ public class MainApplication {
|
||||
sa.run(args);
|
||||
LOGGER.info("MainApplication started");
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
LOGGER.error("start failed and application exit", e);
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ import java.util.stream.Collectors;
|
||||
*/
|
||||
@Api(tags = "开放接口-Broker相关接口(REST)")
|
||||
@RestController
|
||||
@RequestMapping(ApiPrefix.API_V1_THIRD_PART_OP_PREFIX)
|
||||
@RequestMapping(ApiPrefix.API_V1_THIRD_PART_PREFIX)
|
||||
public class ThirdPartBrokerController {
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
@@ -44,7 +44,7 @@ public class ThirdPartBrokerController {
|
||||
private ClusterService clusterService;
|
||||
|
||||
@ApiOperation(value = "Broker信息概览", notes = "")
|
||||
@RequestMapping(value = "{clusterId}/brokers/{brokerId}/overview", method = RequestMethod.GET)
|
||||
@GetMapping(value = "{clusterId}/brokers/{brokerId}/overview")
|
||||
@ResponseBody
|
||||
public Result<ThirdPartBrokerOverviewVO> getBrokerOverview(@PathVariable Long clusterId,
|
||||
@PathVariable Integer brokerId) {
|
||||
@@ -70,7 +70,7 @@ public class ThirdPartBrokerController {
|
||||
}
|
||||
|
||||
@ApiOperation(value = "BrokerRegion信息", notes = "所有集群的")
|
||||
@RequestMapping(value = "broker-regions", method = RequestMethod.GET)
|
||||
@GetMapping(value = "broker-regions")
|
||||
@ResponseBody
|
||||
public Result<List<BrokerRegionVO>> getBrokerRegions() {
|
||||
List<ClusterDO> clusterDOList = clusterService.list();
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.xiaojukeji.kafka.manager.web.config;
|
||||
|
||||
import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.servlet.config.annotation.*;
|
||||
@@ -20,6 +22,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
|
||||
@EnableWebMvc
|
||||
@EnableSwagger2
|
||||
public class SwaggerConfig implements WebMvcConfigurer {
|
||||
@Autowired
|
||||
private ConfigUtils configUtils;
|
||||
|
||||
@Override
|
||||
public void addResourceHandlers(ResourceHandlerRegistry registry) {
|
||||
registry.addResourceHandler("swagger-ui.html").addResourceLocations("classpath:/META-INF/resources/");
|
||||
@@ -41,7 +46,7 @@ public class SwaggerConfig implements WebMvcConfigurer {
|
||||
return new ApiInfoBuilder()
|
||||
.title("LogiKM接口文档")
|
||||
.description("欢迎使用滴滴LogiKM")
|
||||
.version("2.5.0")
|
||||
.version(configUtils.getApplicationVersion())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -13,11 +13,19 @@ import java.util.List;
|
||||
* @date 19/5/3
|
||||
*/
|
||||
public class AccountConverter {
|
||||
private AccountConverter() {
|
||||
}
|
||||
|
||||
public static AccountDO convert2AccountDO(AccountDTO dto) {
|
||||
AccountDO accountDO = new AccountDO();
|
||||
accountDO.setUsername(dto.getUsername());
|
||||
accountDO.setPassword(dto.getPassword());
|
||||
accountDO.setRole(dto.getRole());
|
||||
|
||||
// 兼容前端未传这些信息的情况
|
||||
accountDO.setDepartment(dto.getDepartment() == null? "": dto.getDepartment());
|
||||
accountDO.setMail(dto.getMail() == null? "": dto.getMail());
|
||||
accountDO.setDisplayName(dto.getDisplayName() == null? "": dto.getDisplayName());
|
||||
return accountDO;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.xiaojukeji.kafka.manager.web.metrics;
|
||||
|
||||
import com.codahale.metrics.*;
|
||||
import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
|
||||
import com.xiaojukeji.kafka.manager.common.utils.factory.DefaultThreadFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -21,7 +20,7 @@ import java.util.concurrent.TimeUnit;
|
||||
*/
|
||||
@Component
|
||||
public class MetricsRegistry {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.API_METRICS_LOGGER);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MetricsRegistry.class);
|
||||
|
||||
private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("#");
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ server:
|
||||
spring:
|
||||
application:
|
||||
name: kafkamanager
|
||||
version: @project.version@
|
||||
profiles:
|
||||
active: dev
|
||||
datasource:
|
||||
@@ -32,7 +33,6 @@ custom:
|
||||
idc: cn
|
||||
store-metrics-task:
|
||||
community:
|
||||
broker-metrics-enabled: true
|
||||
topic-metrics-enabled: true
|
||||
didi: # 滴滴Kafka特有的指标
|
||||
app-topic-metrics-enabled: false
|
||||
@@ -42,13 +42,20 @@ custom:
|
||||
# 任务相关的配置
|
||||
task:
|
||||
op:
|
||||
sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
|
||||
order-auto-exec: # 工单自动化审批线程的开关
|
||||
topic-enabled: false # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
app-enabled: false # App工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
|
||||
order-auto-exec: # 工单自动化审批线程的开关
|
||||
topic-enabled: false # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
app-enabled: false # App工单自动化审批开关, false:关闭自动化审批, true:开启
|
||||
metrics:
|
||||
delete-metrics:
|
||||
delete-limit-size: 1000
|
||||
collect: # 收集指标
|
||||
broker-metrics-enabled: true # 收集Broker指标
|
||||
sink: # 上报指标
|
||||
cluster-metrics: # 上报cluster指标
|
||||
sink-db-enabled: true # 上报到db
|
||||
broker-metrics: # 上报broker指标
|
||||
sink-db-enabled: true # 上报到db
|
||||
delete: # 删除指标
|
||||
delete-limit-size: 1000 # 单次删除的批大小
|
||||
cluster-metrics-save-days: 14 # 集群指标保存天数
|
||||
broker-metrics-save-days: 14 # Broker指标保存天数
|
||||
topic-metrics-save-days: 7 # Topic指标保存天数
|
||||
@@ -56,7 +63,25 @@ task:
|
||||
topic-throttled-metrics-save-days: 7 # Topic限流指标保存天数
|
||||
app-topic-metrics-save-days: 7 # App+Topic指标保存天数
|
||||
|
||||
thread-pool:
|
||||
collect-metrics:
|
||||
thread-num: 256 # 收集指标线程池大小
|
||||
queue-size: 5000 # 收集指标线程池的queue大小
|
||||
api-call:
|
||||
thread-num: 16 # api服务线程池大小
|
||||
queue-size: 5000 # api服务线程池的queue大小
|
||||
|
||||
client-pool:
|
||||
kafka-consumer:
|
||||
min-idle-client-num: 24 # 最小空闲客户端数
|
||||
max-idle-client-num: 24 # 最大空闲客户端数
|
||||
max-total-client-num: 24 # 最大客户端数
|
||||
borrow-timeout-unit-ms: 3000 # 租借超时时间,单位毫秒
|
||||
|
||||
account:
|
||||
jump-login:
|
||||
gateway-api: false # 网关接口
|
||||
third-part-api: false # 第三方接口
|
||||
ldap:
|
||||
enabled: false
|
||||
url: ldap://127.0.0.1:389/
|
||||
@@ -70,19 +95,20 @@ account:
|
||||
auth-user-registration: true
|
||||
auth-user-registration-role: normal
|
||||
|
||||
kcm:
|
||||
enabled: false
|
||||
s3:
|
||||
kcm: # 集群安装部署,仅安装broker
|
||||
enabled: false # 是否开启
|
||||
s3: # s3 存储服务
|
||||
endpoint: s3.didiyunapi.com
|
||||
access-key: 1234567890
|
||||
secret-key: 0987654321
|
||||
bucket: logi-kafka
|
||||
n9e:
|
||||
base-url: http://127.0.0.1:8004
|
||||
user-token: 12345678
|
||||
timeout: 300
|
||||
account: root
|
||||
script-file: kcm_script.sh
|
||||
n9e: # 夜莺
|
||||
base-url: http://127.0.0.1:8004 # 夜莺job服务地址
|
||||
user-token: 12345678 # 用户的token
|
||||
timeout: 300 # 当台操作的超时时间
|
||||
account: root # 操作时使用的账号
|
||||
script-file: kcm_script.sh # 脚本,已内置好,在源码的kcm模块内,此处配置无需修改
|
||||
logikm-url: http://127.0.0.1:8080 # logikm部署地址,部署时kcm_script.sh会调用logikm检查部署中的一些状态
|
||||
|
||||
monitor:
|
||||
enabled: false
|
||||
@@ -102,18 +128,3 @@ notify:
|
||||
topic-name: didi-kafka-notify
|
||||
order:
|
||||
detail-url: http://127.0.0.1
|
||||
|
||||
thread-pool:
|
||||
collect-metrics:
|
||||
thread-num: 256 # 收集指标线程池大小
|
||||
queue-size: 5000 # 收集指标线程池的queue大小
|
||||
api-call:
|
||||
thread-num: 16 # api服务线程池大小
|
||||
queue-size: 5000 # api服务线程池的queue大小
|
||||
|
||||
client-pool:
|
||||
kafka-consumer:
|
||||
min-idle-client-num: 24 # 最小空闲客户端数
|
||||
max-idle-client-num: 24 # 最大空闲客户端数
|
||||
max-total-client-num: 24 # 最大客户端数
|
||||
borrow-timeout-unit-ms: 3000 # 租借超时时间,单位秒
|
||||
|
||||
@@ -131,15 +131,15 @@
|
||||
</filter>
|
||||
</appender>
|
||||
|
||||
<!-- Metrics信息收集日志 -->
|
||||
<appender name="COLLECTOR_METRICS_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.path}/metrics/collector_metrics.log</file>
|
||||
<!-- Task模块相关日志 -->
|
||||
<appender name="TASK_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.path}/log_task.log</file>
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
|
||||
<charset>UTF-8</charset>
|
||||
</encoder>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${log.path}/metrics/collector_metrics_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
|
||||
<fileNamePattern>${log.path}/log_task_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>100MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
@@ -147,15 +147,15 @@
|
||||
</rollingPolicy>
|
||||
</appender>
|
||||
|
||||
<!-- Metrics信息收集日志 -->
|
||||
<!-- Api-Metrics信息相关日志 -->
|
||||
<appender name="API_METRICS_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.path}/metrics/api_metrics.log</file>
|
||||
<file>${log.path}/api_metrics.log</file>
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
|
||||
<charset>UTF-8</charset>
|
||||
</encoder>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${log.path}/metrics/api_metrics_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
|
||||
<fileNamePattern>${log.path}/api_metrics_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>100MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
@@ -163,31 +163,13 @@
|
||||
</rollingPolicy>
|
||||
</appender>
|
||||
|
||||
<!-- Metrics信息收集日志 -->
|
||||
<appender name="SCHEDULED_TASK_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.path}/metrics/scheduled_tasks.log</file>
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
|
||||
<charset>UTF-8</charset>
|
||||
</encoder>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${log.path}/metrics/scheduled_tasks_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>100MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
<maxHistory>5</maxHistory>
|
||||
</rollingPolicy>
|
||||
</appender>
|
||||
<logger name="com.xiaojukeji.kafka.manager.task" level="INFO" additivity="false">
|
||||
<appender-ref ref="TASK_LOGGER"/>
|
||||
</logger>
|
||||
|
||||
<logger name="COLLECTOR_METRICS_LOGGER" level="DEBUG" additivity="false">
|
||||
<appender-ref ref="COLLECTOR_METRICS_LOGGER"/>
|
||||
</logger>
|
||||
<logger name="API_METRICS_LOGGER" level="DEBUG" additivity="false">
|
||||
<logger name="com.xiaojukeji.kafka.manager.web.metrics" level="INFO" additivity="false">
|
||||
<appender-ref ref="API_METRICS_LOGGER"/>
|
||||
</logger>
|
||||
<logger name="SCHEDULED_TASK_LOGGER" level="DEBUG" additivity="false">
|
||||
<appender-ref ref="SCHEDULED_TASK_LOGGER"/>
|
||||
</logger>
|
||||
|
||||
<logger name="org.apache.ibatis" level="INFO" additivity="false" />
|
||||
<logger name="org.mybatis.spring" level="INFO" additivity="false" />
|
||||
@@ -199,17 +181,6 @@
|
||||
<appender-ref ref="INFO_FILE" />
|
||||
<appender-ref ref="WARN_FILE" />
|
||||
<appender-ref ref="ERROR_FILE" />
|
||||
<!--<appender-ref ref="METRICS_LOG" />-->
|
||||
<!-- <appender-ref ref="TASK_LOGGER" />-->
|
||||
</root>
|
||||
|
||||
<!--生产环境:输出到文件-->
|
||||
<!--<springProfile name="pro">-->
|
||||
<!--<root level="info">-->
|
||||
<!--<appender-ref ref="CONSOLE" />-->
|
||||
<!--<appender-ref ref="DEBUG_FILE" />-->
|
||||
<!--<appender-ref ref="INFO_FILE" />-->
|
||||
<!--<appender-ref ref="ERROR_FILE" />-->
|
||||
<!--<appender-ref ref="WARN_FILE" />-->
|
||||
<!--</root>-->
|
||||
<!--</springProfile>-->
|
||||
</configuration>
|
||||
10
pom.xml
10
pom.xml
@@ -11,12 +11,13 @@
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.1.1.RELEASE</version> <!-- 莫要修改该版本, 否则可能会导致自自定义的schedule不生效 -->
|
||||
<version>2.1.18.RELEASE</version> <!-- 莫要修改该版本, 否则可能会导致自自定义的schedule不生效 -->
|
||||
<!--<relativePath/>-->
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<kafka-manager.revision>2.5</kafka-manager.revision>
|
||||
<kafka-manager.revision>2.6.0</kafka-manager.revision>
|
||||
<spring.boot.version>2.1.18.RELEASE</spring.boot.version>
|
||||
<swagger2.version>2.9.2</swagger2.version>
|
||||
<swagger.version>1.5.21</swagger.version>
|
||||
|
||||
@@ -65,6 +66,11 @@
|
||||
<artifactId>swagger-annotations</artifactId>
|
||||
<version>${swagger.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.swagger</groupId>
|
||||
<artifactId>swagger-models</artifactId>
|
||||
<version>${swagger.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- javax -->
|
||||
<dependency>
|
||||
|
||||
Reference in New Issue
Block a user