Merge branch 'dev' into dev_v2.6.0

README.md

@@ -5,7 +5,7 @@
 
 **一站式`Apache Kafka`集群指标监控与运维管控平台**
 
-
+`LogiKM开源至今备受关注，考虑到开源项目应该更贴合Apache Kafka未来发展方向，经项目组慎重考虑，预计22年5月份将其品牌升级成Know Streaming，届时项目名称和Logo也将统一更新，感谢大家一如既往的支持，敬请期待！`
 
 阅读本README文档，您可以了解到滴滴Logi-KafkaManager的用户群体、产品定位等信息，并通过体验地址，快速体验Kafka集群指标监控与运维管控的全流程。
 
@@ -55,39 +55,63 @@
 ## 2 相关文档
 
 ### 2.1 产品文档
-- [滴滴Logi-KafkaManager 安装手册](docs/install_guide/install_guide_cn.md)
+- [滴滴LogiKM 安装手册](docs/install_guide/install_guide_cn.md)
-- [滴滴Logi-KafkaManager 接入集群](docs/user_guide/add_cluster/add_cluster.md)
+- [滴滴LogiKM 接入集群](docs/user_guide/add_cluster/add_cluster.md)
-- [滴滴Logi-KafkaManager 用户使用手册](docs/user_guide/user_guide_cn.md)
+- [滴滴LogiKM 用户使用手册](docs/user_guide/user_guide_cn.md)
-- [滴滴Logi-KafkaManager FAQ](docs/user_guide/faq.md)
+- [滴滴LogiKM FAQ](docs/user_guide/faq.md)
 
 ### 2.2 社区文章
+- [kafka最强最全知识图谱](https://www.szzdzhp.com/kafka/)
+- [LogiKM新用户入门系列文章专栏 --石臻臻](https://www.szzdzhp.com/categories/LogIKM/)
 - [滴滴云官网产品介绍](https://www.didiyun.com/production/logi-KafkaManager.html)
 - [7年沉淀之作--滴滴Logi日志服务套件](https://mp.weixin.qq.com/s/-KQp-Qo3WKEOc9wIR2iFnw)
 - [滴滴Logi-KafkaManager 一站式Kafka监控与管控平台](https://mp.weixin.qq.com/s/9qSZIkqCnU6u9nLMvOOjIQ)
 - [滴滴Logi-KafkaManager 开源之路](https://xie.infoq.cn/article/0223091a99e697412073c0d64)
-- [滴滴Logi-KafkaManager 系列视频教程](https://mp.weixin.qq.com/s/9X7gH0tptHPtfjPPSdGO8g)
+- [滴滴Logi-KafkaManager 系列视频教程](https://space.bilibili.com/442531657/channel/seriesdetail?sid=571649)
 - [kafka实践（十五）：滴滴开源Kafka管控平台 Logi-KafkaManager研究--A叶子叶来](https://blog.csdn.net/yezonggang/article/details/113106244)
-- [kafka的灵魂伴侣Logi-KafkaManager系列文章专栏 --石臻](https://blog.csdn.net/u010634066/category_10977588.html)
+
 
 ## 3 滴滴Logi开源用户交流群
 
-
 ![image](https://user-images.githubusercontent.com/5287750/111266722-e531d800-8665-11eb-9242-3484da5a3099.png)  
-微信加群：添加mike_zhangliang的微信号备注Logi加群或关注公众号 云原生可观测性 回复 "Logi加群"
+
+想跟各个大佬交流Kafka Es 等中间件/大数据相关技术请 加微信进群。
+
+微信加群：添加<font color=red>mike_zhangliang</font>、<font color=red>danke-xie</font>的微信号备注Logi加群或关注公众号 云原生可观测性 回复 "Logi加群"
+
+## 4 知识星球
+
+<img width="447" alt="image" src="https://user-images.githubusercontent.com/71620349/147314042-843a371a-48c0-4d9a-a65e-ca40236f3300.png">
+
+<br>
+<center>
+  ✅我们正在组建国内最大最权威的
+</center>
+<br>
+<center>
+<font color=red size=5><b>【Kafka中文社区】</b></font>
+</center>
+  
+在这里你可以结交各大互联网Kafka大佬以及近2000+Kafka爱好者，一起实现知识共享，实时掌控最新行业资讯，期待您的加入中～https://z.didi.cn/5gSF9  
+
+<font color=red size=5>有问必答~！  </font>
+
+<font color=red size=5>互动有礼~！  </font>
+
+PS:提问请尽量把问题一次性描述清楚，并告知环境信息情况哦～！如使用版本、操作步骤、报错/警告信息等，方便大V们快速解答～
+
+## 5 项目成员
+
+### 5.1 内部核心人员
+
+`iceyuhui`、`liuyaguang`、`limengmonty`、`zhangliangmike`、`xiepeng`、`nullhuangyiming`、`zengqiao`、`eilenexuzhe`、`huangjiaweihjw`、`zhaoyinrui`、`marzkonglingxu`、`joysunchao`、`石臻臻`
 
 
-## 4 项目成员
+### 5.2 外部贡献者
-
-### 4.1 内部核心人员
-
-`iceyuhui`、`liuyaguang`、`limengmonty`、`zhangliangmike`、`nullhuangyiming`、`zengqiao`、`eilenexuzhe`、`huangjiaweihjw`、`zhaoyinrui`、`marzkonglingxu`、`joysunchao`
-
-
-### 4.2 外部贡献者
 
 `fangjunyu`、`zhoutaiyang`
 
 
-## 5 协议
+## 6 协议
 
-`kafka-manager`基于`Apache-2.0`协议进行分发和使用，更多信息参见[协议文件](./LICENSE)
+`LogiKM`基于`Apache-2.0`协议进行分发和使用，更多信息参见[协议文件](./LICENSE)

build.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-workspace=$(cd $(dirname $0) && pwd -P)
-cd $workspace
-
-## constant
-OUTPUT_DIR=./output
-KM_VERSION=2.4.2
-APP_NAME=kafka-manager
-APP_DIR=${APP_NAME}-${KM_VERSION}
-
-MYSQL_TABLE_SQL_FILE=./docs/install_guide/create_mysql_table.sql
-CONFIG_FILE=./kafka-manager-web/src/main/resources/application.yml
-
-## function
-function build() {
-    # 编译命令
-	  mvn -U clean package -Dmaven.test.skip=true
-
-    local sc=$?
-    if [ $sc -ne 0 ];then
-    	## 编译失败, 退出码为 非0
-        echo "$APP_NAME build error"
-        exit $sc
-    else
-        echo "$APP_NAME build ok"
-    fi
-}
-
-function make_output() {
-    # 新建output目录
-    rm -rf ${OUTPUT_DIR} &>/dev/null
-    mkdir -p ${OUTPUT_DIR}/${APP_DIR} &>/dev/null
-
-    # 填充output目录, output内的内容
-    (
-          cp -rf ${MYSQL_TABLE_SQL_FILE} ${OUTPUT_DIR}/${APP_DIR} &&       # 拷贝 sql 初始化脚本      至output目录
-          cp -rf ${CONFIG_FILE} ${OUTPUT_DIR}/${APP_DIR} &&                # 拷贝 application.yml   至output目录
-
-          # 拷贝程序包到output路径
-          cp kafka-manager-web/target/kafka-manager-web-${KM_VERSION}-SNAPSHOT.jar ${OUTPUT_DIR}/${APP_DIR}/${APP_NAME}.jar
-          echo -e "make output ok."
-    ) || { echo -e "make output error"; exit 2; } # 填充output目录失败后, 退出码为 非0
-}
-
-function make_package() {
-	# 压缩output目录
-	(
-	    cd ${OUTPUT_DIR} && tar cvzf ${APP_DIR}.tar.gz ${APP_DIR}
-      echo -e "make package ok."
-	) || { echo -e "make package error"; exit 2; } # 压缩output目录失败后, 退出码为 非0
-}
-
-##########################################
-## main
-## 其中,
-## 		1.进行编译
-##		2.生成部署包output
-##		3.生成tar.gz压缩包
-##########################################
-
-# 1.进行编译
-build
-
-# 2.生成部署包output
-make_output
-
-# 3.生成tar.gz压缩包
-make_package
-
-# 编译成功
-echo -e "build done"
-exit 0

container/dockerfiles/Dockerfile

@@ -19,6 +19,7 @@ ENV JAVA_OPTS="-verbose:gc  \
 RUN wget https://github.com/didi/Logi-KafkaManager/releases/download/v${VERSION}/kafka-manager-${VERSION}.tar.gz && \
     tar xvf kafka-manager-${VERSION}.tar.gz && \
     mv kafka-manager-${VERSION}/kafka-manager.jar /opt/app.jar && \
+    mv kafka-manager-${VERSION}/application.yml /opt/application.yml && \
     rm -rf kafka-manager-${VERSION}*
 
 EXPOSE 8080  9999

container/helm/templates/configmap.yaml

@@ -55,7 +55,7 @@ data:
         didi:
           app-topic-metrics-enabled: false
           topic-request-time-metrics-enabled: false
-          topic-throttled-metrics: false
+          topic-throttled-metrics-enabled: false
         save-days: 7
 
     # 任务相关的开关
@@ -67,7 +67,16 @@ data:
       # ldap settings
       ldap:
         enabled: false
-        authUserRegistration: false
+        url: ldap://127.0.0.1:389/
+        basedn: dc=tsign,dc=cn
+        factory: com.sun.jndi.ldap.LdapCtxFactory
+        filter: sAMAccountName
+        security:
+          authentication: simple
+          principal: cn=admin,dc=tsign,dc=cn
+          credentials: admin
+        auth-user-registration: false
+        auth-user-registration-role: normal
 
     kcm:
       enabled: false

distribution/bin/shutdown.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd `dirname $0`/../target
+target_dir=`pwd`
+
+pid=`ps ax | grep -i 'kafka-manager' | grep ${target_dir} | grep java | grep -v grep | awk '{print $1}'`
+if [ -z "$pid" ] ; then
+        echo "No kafka-manager running."
+        exit -1;
+fi
+
+echo "The kafka-manager (${pid}) is running..."
+
+kill ${pid}
+
+echo "Send shutdown request to kafka-manager (${pid}) OK"

distribution/bin/startup.sh

@@ -0,0 +1,81 @@
+error_exit ()
+{
+    echo "ERROR: $1 !!"
+    exit 1
+}
+
+[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java
+[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/usr/java
+[ ! -e "$JAVA_HOME/bin/java" ] && unset JAVA_HOME
+
+if [ -z "$JAVA_HOME" ]; then
+  if $darwin; then
+
+    if [ -x '/usr/libexec/java_home' ] ; then
+      export JAVA_HOME=`/usr/libexec/java_home`
+
+    elif [ -d "/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" ]; then
+      export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home"
+    fi
+  else
+    JAVA_PATH=`dirname $(readlink -f $(which javac))`
+    if [ "x$JAVA_PATH" != "x" ]; then
+      export JAVA_HOME=`dirname $JAVA_PATH 2>/dev/null`
+    fi
+  fi
+  if [ -z "$JAVA_HOME" ]; then
+        error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)! jdk8 or later is better!"
+  fi
+fi
+
+
+
+
+export WEB_SERVER="kafka-manager"
+export JAVA_HOME
+export JAVA="$JAVA_HOME/bin/java"
+export BASE_DIR=`cd $(dirname $0)/..; pwd`
+export CUSTOM_SEARCH_LOCATIONS=file:${BASE_DIR}/conf/
+
+
+#===========================================================================================
+# JVM Configuration
+#===========================================================================================
+
+JAVA_OPT="${JAVA_OPT} -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m"
+JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof"
+
+## jdk版本高的情况 有些 参数废弃了
+JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
+if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
+  JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/km_gc.log:time,tags:filecount=10,filesize=102400"
+else
+  JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext"
+  JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/km_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
+
+fi
+
+JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/${WEB_SERVER}.jar"
+JAVA_OPT="${JAVA_OPT} --spring.config.additional-location=${CUSTOM_SEARCH_LOCATIONS}"
+JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/logback-spring.xml"
+JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288"
+
+
+
+if [ ! -d "${BASE_DIR}/logs" ]; then
+  mkdir ${BASE_DIR}/logs
+fi
+
+echo "$JAVA ${JAVA_OPT}"
+
+# check the start.out log output file
+if [ ! -f "${BASE_DIR}/logs/start.out" ]; then
+  touch "${BASE_DIR}/logs/start.out"
+fi
+# start
+echo -e "---- 启动脚本 ------\n $JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 &
+
+
+nohup $JAVA ${JAVA_OPT} >> ${BASE_DIR}/logs/start.out 2>&1 &
+
+echo "${WEB_SERVER} is starting，you can check the ${BASE_DIR}/logs/start.out"

distribution/conf/application.yml

@@ -0,0 +1,28 @@
+
+## kafka-manager的配置文件，该文件中的配置会覆盖默认配置
+## 下面的配置信息基本就是jar中的 application.yml默认配置了;
+## 可以只修改自己变更的配置,其他的删除就行了; 比如只配置一下mysql
+
+
+server:
+  port: 8080
+  tomcat:
+    accept-count: 1000
+    max-connections: 10000
+    max-threads: 800
+    min-spare-threads: 100
+
+spring:
+  application:
+    name: kafkamanager
+  profiles:
+    active: dev
+  datasource:
+    kafka-manager:
+      jdbc-url: jdbc:mysql://localhost:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8
+      username: root
+      password: 123456
+      driver-class-name: com.mysql.cj.jdbc.Driver
+  main:
+    allow-bean-definition-overriding: true
+

distribution/conf/application.yml.example

@@ -0,0 +1,124 @@
+
+## kafka-manager的配置文件，该文件中的配置会覆盖默认配置
+## 下面的配置信息基本就是jar中的 application.yml默认配置了;
+## 可以只修改自己变更的配置,其他的删除就行了; 比如只配置一下mysql
+
+
+server:
+  port: 8080
+  tomcat:
+    accept-count: 1000
+    max-connections: 10000
+    max-threads: 800
+    min-spare-threads: 100
+
+spring:
+  application:
+    name: kafkamanager
+  profiles:
+    active: dev
+  datasource:
+    kafka-manager:
+      jdbc-url: jdbc:mysql://localhost:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8
+      username: root
+      password: 123456
+      driver-class-name: com.mysql.cj.jdbc.Driver
+  main:
+    allow-bean-definition-overriding: true
+
+
+  servlet:
+    multipart:
+      max-file-size: 100MB
+      max-request-size: 100MB
+
+logging:
+  config: classpath:logback-spring.xml
+
+custom:
+  idc: cn   # 部署的数据中心, 忽略该配置, 后续会进行删除
+  jmx:
+    max-conn: 10 # 2.3版本配置不在这个地方生效
+  store-metrics-task:
+    community:
+      broker-metrics-enabled: true # 社区部分broker metrics信息收集开关, 关闭之后metrics信息将不会进行收集及写DB
+      topic-metrics-enabled: true  # 社区部分topic的metrics信息收集开关, 关闭之后metrics信息将不会进行收集及写DB
+    didi:
+      app-topic-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
+      topic-request-time-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
+      topic-throttled-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
+
+# 任务相关的开关
+task:
+  op:
+    sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
+    order-auto-exec:          # 工单自动化审批线程的开关
+      topic-enabled: false      # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
+      app-enabled: false        # App工单自动化审批开关, false:关闭自动化审批, true:开启
+  metrics:
+    delete-metrics:
+      delete-limit-size: 1000
+      cluster-metrics-save-days: 14                   # 集群指标保存天数
+      broker-metrics-save-days: 14                    # Broker指标保存天数
+      topic-metrics-save-days: 7                      # Topic指标保存天数
+      topic-request-time-metrics-save-days: 7         # Topic请求耗时指标保存天数
+      topic-throttled-metrics-save-days: 7            # Topic限流指标保存天数
+      app-topic-metrics-save-days: 7                  # App+Topic指标保存天数
+
+# ldap相关的配置
+account:
+  ldap:
+    enabled: false
+    url: ldap://127.0.0.1:389/
+    basedn: dc=tsign,dc=cn
+    factory: com.sun.jndi.ldap.LdapCtxFactory
+    filter: sAMAccountName
+    security:
+      authentication: simple
+      principal: cn=admin,dc=tsign,dc=cn
+      credentials: admin
+    auth-user-registration: true
+    auth-user-registration-role: normal
+
+# 集群升级部署相关的功能，需要配合夜莺及S3进行使用
+kcm:
+  enabled: false
+  s3:
+    endpoint: s3.didiyunapi.com
+    access-key: 1234567890
+    secret-key: 0987654321
+    bucket: logi-kafka
+  n9e:
+    base-url: http://127.0.0.1:8004
+    user-token: 12345678
+    timeout: 300
+    account: root
+    script-file: kcm_script.sh
+
+# 监控告警相关的功能，需要配合夜莺进行使用
+# enabled: 表示是否开启监控告警的功能, true: 开启, false: 不开启
+# n9e.nid: 夜莺的节点ID
+# n9e.user-token: 用户的密钥，在夜莺的个人设置中
+# n9e.mon.base-url: 监控地址
+# n9e.sink.base-url: 数据上报地址
+# n9e.rdb.base-url: 用户资源中心地址
+
+monitor:
+  enabled: false
+  n9e:
+    nid: 2
+    user-token: 1234567890
+    mon:
+      base-url: http://127.0.0.1:8000 # 夜莺v4版本，默认端口统一调整为了8000
+    sink:
+      base-url: http://127.0.0.1:8000 # 夜莺v4版本，默认端口统一调整为了8000
+    rdb:
+      base-url: http://127.0.0.1:8000 # 夜莺v4版本，默认端口统一调整为了8000
+
+
+notify: # 通知的功能
+  kafka: # 默认通知发送到kafka的指定Topic中
+    cluster-id: 95 # Topic的集群ID
+    topic-name: didi-kafka-notify # Topic名称
+  order: # 部署的KM的地址
+    detail-url: http://127.0.0.1

distribution/conf/logback-spring.xml

@@ -0,0 +1,215 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration scan="true" scanPeriod="10 seconds">
+    <contextName>logback</contextName>
+    <property name="log.path" value="./logs" />
+
+    <!-- 彩色日志 -->
+    <!-- 彩色日志依赖的渲染类 -->
+    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
+    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
+    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
+    <!-- 彩色日志格式 -->
+    <property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
+
+    <!--输出到控制台-->
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>info</level>
+        </filter>
+        <encoder>
+            <Pattern>${CONSOLE_LOG_PATTERN}</Pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+
+    <!--输出到文件-->
+
+    <!-- 时间滚动输出 level为 DEBUG 日志 -->
+    <appender name="DEBUG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/log_debug.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 日志归档 -->
+            <fileNamePattern>${log.path}/log_debug_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录debug级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>debug</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 INFO 日志 -->
+    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${log.path}/log_info.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 每天日志归档路径以及格式 -->
+            <fileNamePattern>${log.path}/log_info_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录info级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>info</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 WARN 日志 -->
+    <appender name="WARN_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${log.path}/log_warn.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/log_warn_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录warn级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>warn</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+
+    <!-- 时间滚动输出 level为 ERROR 日志 -->
+    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${log.path}/log_error.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/log_error_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录ERROR级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ERROR</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- Metrics信息收集日志 -->
+    <appender name="COLLECTOR_METRICS_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/metrics/collector_metrics.log</file>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/metrics/collector_metrics_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>3</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <!-- Metrics信息收集日志 -->
+    <appender name="API_METRICS_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/metrics/api_metrics.log</file>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/metrics/api_metrics_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>3</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <!-- Metrics信息收集日志 -->
+    <appender name="SCHEDULED_TASK_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/metrics/scheduled_tasks.log</file>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/metrics/scheduled_tasks_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>5</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <logger name="COLLECTOR_METRICS_LOGGER" level="DEBUG" additivity="false">
+        <appender-ref ref="COLLECTOR_METRICS_LOGGER"/>
+    </logger>
+    <logger name="API_METRICS_LOGGER" level="DEBUG" additivity="false">
+        <appender-ref ref="API_METRICS_LOGGER"/>
+    </logger>
+    <logger name="SCHEDULED_TASK_LOGGER" level="DEBUG" additivity="false">
+        <appender-ref ref="SCHEDULED_TASK_LOGGER"/>
+    </logger>
+
+    <logger name="org.apache.ibatis" level="INFO" additivity="false" />
+    <logger name="org.mybatis.spring" level="INFO" additivity="false" />
+    <logger name="com.github.miemiedev.mybatis.paginator" level="INFO" additivity="false" />
+
+    <root level="info">
+        <appender-ref ref="CONSOLE" />
+        <appender-ref ref="DEBUG_FILE" />
+        <appender-ref ref="INFO_FILE" />
+        <appender-ref ref="WARN_FILE" />
+        <appender-ref ref="ERROR_FILE" />
+        <!--<appender-ref ref="METRICS_LOG" />-->
+    </root>
+
+    <!--生产环境:输出到文件-->
+    <!--<springProfile name="pro">-->
+        <!--<root level="info">-->
+            <!--<appender-ref ref="CONSOLE" />-->
+            <!--<appender-ref ref="DEBUG_FILE" />-->
+            <!--<appender-ref ref="INFO_FILE" />-->
+            <!--<appender-ref ref="ERROR_FILE" />-->
+            <!--<appender-ref ref="WARN_FILE" />-->
+        <!--</root>-->
+    <!--</springProfile>-->
+</configuration>

distribution/pom.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <parent>
+        <artifactId>kafka-manager</artifactId>
+        <groupId>com.xiaojukeji.kafka</groupId>
+        <version>${kafka-manager.revision}</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>distribution</artifactId>
+    <name>distribution</name>
+    <packaging>pom</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>kafka-manager-web</artifactId>
+            <version>${kafka-manager.revision}</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+
+        <profile>
+            <id>release-kafka-manager</id>
+            <dependencies>
+                <dependency>
+                    <groupId>${project.groupId}</groupId>
+                    <artifactId>kafka-manager-web</artifactId>
+                    <version>${kafka-manager.revision}</version>
+                </dependency>
+            </dependencies>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>release-km.xml</descriptor>
+                            </descriptors>
+                            <tarLongFileMode>posix</tarLongFileMode>
+                        </configuration>
+                        <executions>
+                            <execution>
+                                <id>make-assembly</id>
+                                <phase>install</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+                <finalName>kafka-manager</finalName>
+            </build>
+        </profile>
+    </profiles>
+</project>

distribution/readme.md

@@ -0,0 +1,22 @@
+## 说明
+
+### 1.创建mysql数据库文件
+> conf/create_mysql_table.sql
+
+### 2. 修改配置文件
+> conf/application.yml.example
+> 请将application.yml.example 复制一份改名为application.yml；
+> 并放在同级目录下(conf/); 并修改成自己的配置
+> 这里的优先级比jar包内配置文件的默认值高; 
+> 
+
+### 3.启动/关闭kafka-manager
+> sh bin/startup.sh    启动
+> 
+> sh shutdown.sh       关闭
+> 
+
+
+### 4.升级jar包
+> 如果是升级, 可以看看文件 `upgrade_config.md` 的配置变更历史;
+> 

distribution/release-km.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<assembly>
+    <id>${project.version}</id>
+    <includeBaseDirectory>true</includeBaseDirectory>
+    <formats>
+        <format>dir</format>
+        <format>tar.gz</format>
+        <format>zip</format>
+    </formats>
+    <fileSets>
+        <fileSet>
+            <includes>
+                <include>conf/**</include>
+            </includes>
+        </fileSet>
+
+        <fileSet>
+            <includes>
+                <include>bin/*</include>
+            </includes>
+            <fileMode>0755</fileMode>
+        </fileSet>
+    </fileSets>
+    <files>
+
+
+        <file>
+            <source>readme.md</source>
+            <destName>readme.md</destName>
+        </file>
+        <file>
+            <source>upgrade_config.md</source>
+            <destName>upgrade_config.md</destName>
+        </file>
+        <file>
+            <!--打好的jar包名称和放置目录-->
+            <source>../kafka-manager-web/target/kafka-manager.jar</source>
+            <outputDirectory>target/</outputDirectory>
+        </file>
+    </files>
+
+    <moduleSets>
+        <moduleSet>
+            <useAllReactorProjects>true</useAllReactorProjects>
+            <includes>
+                <include>com.xiaojukeji.kafka:kafka-manager-web</include>
+            </includes>
+        </moduleSet>
+    </moduleSets>
+</assembly>

distribution/upgrade_config.md

@@ -0,0 +1,42 @@
+
+## 版本升级配置变更
+> 本文件 从 V2.2.0 开始记录; 如果配置有变更则会填写到下文中; 如果没有,则表示无变更;
+> 当您从一个很低的版本升级时候,应该依次执行中间有过变更的sql脚本
+
+
+
+**一站式`Apache Kafka`集群指标监控与运维管控平台**
+
+--- 
+
+### 1.升级至`V2.2.0`版本
+
+#### 1.mysql变更
+
+`2.2.0`版本在`cluster`表及`logical_cluster`各增加了一个字段，因此需要执行下面的sql进行字段的增加。
+
+```sql
+# 往cluster表中增加jmx_properties字段, 这个字段会用于存储jmx相关的认证以及配置信息
+ALTER TABLE `cluster` ADD COLUMN `jmx_properties` TEXT NULL COMMENT 'JMX配置' AFTER `security_properties`;
+
+# 往logical_cluster中增加identification字段, 同时数据和原先name数据相同, 最后增加一个唯一键. 
+# 此后, name字段还是表示集群名称, 而identification字段表示的是集群标识, 只能是字母数字及下划线组成, 
+# 数据上报到监控系统时, 集群这个标识采用的字段就是identification字段, 之前使用的是name字段.
+ALTER TABLE `logical_cluster` ADD COLUMN `identification` VARCHAR(192) NOT NULL DEFAULT '' COMMENT '逻辑集群标识' AFTER `name`;
+
+UPDATE `logical_cluster` SET `identification`=`name` WHERE id>=0;
+
+ALTER TABLE `logical_cluster` ADD INDEX `uniq_identification` (`identification` ASC);
+```
+
+### 升级至`2.3.0`版本
+
+#### 1.mysql变更
+`2.3.0`版本在`gateway_config`表增加了一个描述说明的字段，因此需要执行下面的sql进行字段的增加。
+
+```sql
+ALTER TABLE `gateway_config` 
+ADD COLUMN `description` TEXT NULL COMMENT '描述信息' AFTER `version`;
+```
+
+

docs/dev_guide/如何增加上报监控系统指标.md

@@ -0,0 +1,53 @@
+
+---
+
+![kafka-manager-logo](../assets/images/common/logo_name.png)
+
+**一站式`Apache Kafka`集群指标监控与运维管控平台**
+
+--- 
+
+# 如何增加上报监控系统指标？
+
+## 0、前言
+
+LogiKM是 **一站式`Apache Kafka`集群指标监控与运维管控平台** ，当前会将消费Lag，Topic流量等指标上报到监控系统中，从而方便用户在监控系统中对这些指标配置监控告警规则，进而达到监控自身客户端是否正常的目的。
+
+那么，如果我们想增加一个新的监控指标，应该如何做呢，比如我们想监控Broker的流量，监控Broker的存活信息，监控集群Controller个数等等。
+
+在具体介绍之前，我们大家都知道，Kafka监控相关的信息，基本都存储于Broker、Jmx以及ZK中。当前LogiKM也已经具备从这三个地方获取数据的基本能力，因此基于LogiKM我们再获取其他指标，总体上还是非常方便的。
+
+这里我们就以已经获取到的Topic流量信息为例，看LogiKM如何实现Topic指标的获取并上报的。
+
+---
+
+## 1、确定指标位置
+
+基于对Kafka的了解，我们知道Topic流量信息这个指标是存储于Jmx中的，因此我们需要从Jmx中获取。大家如果对于自己所需要获取的指标存储在何处不太清楚的，可以加入我们维护的Kafka中文社区(README中有二维码)中今天沟通交流。
+
+---
+
+## 2、指标获取
+
+Topic流量指标的获取详细见图中说明。
+
+![Topic流量指标采集说明](./assets/increase_the_indicators_reported_to_monitor_system/collect_topic_metrics.jpg)
+
+---
+
+## 3、指标上报
+
+上一步我们已经采集到Topic流量指标了，下一步就是将该指标上报到监控系统，这块只需要按照监控系统要求的格式，将数据上报即可。
+
+LogiKM中有一个monitor模块，具体的如下图所示：
+
+![指标上报](./assets/increase_the_indicators_reported_to_monitor_system/sink_metrcis.png)
+
+
+## 4、补充说明 
+
+监控系统对接的相关内容见：
+
+[监控系统集成](./monitor_system_integrate_with_self.md)
+
+[监控系统集成例子——集成夜莺](./monitor_system_integrate_with_n9e.md)

docs/install_guide/config_description.md

@@ -51,13 +51,16 @@ custom:
     didi:
       app-topic-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
       topic-request-time-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
-      topic-throttled-metrics: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
+      topic-throttled-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭
     save-days: 7 #指标在DB中保持的天数，-1表示永久保存，7表示保存近7天的数据
 
 # 任务相关的开关
 task:
   op:
     sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
+    order-auto-exec:          # 工单自动化审批线程的开关
+      topic-enabled: false      # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
+      app-enabled: false        # App工单自动化审批开关, false:关闭自动化审批, true:开启
 
 account: # ldap相关的配置, 社区版本暂时支持不够完善，可以先忽略，欢迎贡献代码对这块做优化
   ldap:

docs/install_guide/install_guide_cn.md

@@ -31,17 +31,23 @@
 
 **2、源代码进行打包**
 
-下载好代码之后，进入`Logi-KafkaManager`的主目录，执行`sh build.sh`命令即可，执行完成之后会在`output/kafka-manager-xxx`目录下面生成一个jar包。
+下载好代码之后，进入`Logi-KafkaManager`的主目录，执行`mvn -Prelease-kafka-manager -Dmaven.test.skip=true clean install -U  `命令即可，
+执行完成之后会在`distribution/target`目录下面生成一个`kafka-manager-*.tar.gz`。
+和一个`kafka-manager-*.zip` 文件,随便任意一个压缩包都可以; 
+当然此时同级目录有一个已经解压好的文件夹;
 
-对于`windows`环境的用户，估计执行不了`sh build.sh`命令，因此可以直接执行`mvn install`，然后在`kafka-manager-web/target`目录下生成一个kafka-manager-web-xxx.jar的包。
 
-获取到jar包之后，我们继续下面的步骤。
 
 ---
 
-## 3、MySQL-DB初始化
+## 3. 解压安装包
+解压完成后; 在文件目录中可以看到有`kafka-manager/conf/create_mysql_table.sql` 有个mysql初始化文件
+先初始化DB
 
-执行[create_mysql_table.sql](create_mysql_table.sql)中的SQL命令，从而创建所需的MySQL库及表，默认创建的库名是`logi_kafka_manager`。
+      
+## 4、MySQL-DB初始化
+
+执行[create_mysql_table.sql](../../distribution/conf/create_mysql_table.sql)中的SQL命令，从而创建所需的MySQL库及表，默认创建的库名是`logi_kafka_manager`。
 
 ```
 # 示例：
@@ -50,15 +56,38 @@ mysql -uXXXX -pXXX -h XXX.XXX.XXX.XXX -PXXXX < ./create_mysql_table.sql
 
 ---
 
-## 4、启动
+## 5.修该配置
+请将`conf/application.yml.example` 文件复制一份出来命名为`application.yml` 放在同级目录:conf/application.yml ;
+并且修改配置; 当然不修改的话 就会用默认的配置;
+至少 mysql配置成自己的吧
 
-```
-# application.yml 是配置文件，最简单的是仅修改MySQL相关的配置即可启动
 
-nohup java -jar kafka-manager.jar --spring.config.location=./application.yml > /dev/null 2>&1 &
+## 6、启动/关闭
-```
+解压包中有启动和关闭脚本 
+`kafka-manager/bin/shutdown.sh`
+`kafka-manager/bin/startup.sh`
 
-### 5、使用
+执行 sh startup.sh 启动
+执行 sh shutdown.sh 关闭
+
+
+
+### 6、使用
 
 本地启动的话，访问`http://localhost:8080`，输入帐号及密码(默认`admin/admin`)进行登录。更多参考：[kafka-manager 用户使用手册](../user_guide/user_guide_cn.md)
 
+### 7. 升级
+
+如果是升级版本,请查看文件 [kafka-manager 升级手册](../../distribution/upgrade_config.md) 
+ 在您下载的启动包(V2.5及其后)中也有记录,在 kafka-manager/upgrade_config.md 中
+
+
+### 8. 在IDE中启动
+> 如果想参与开发或者想在IDE中启动的话
+> 先执行  `mvn -Dmaven.test.skip=true clean install -U ` 
+> 
+> 然后这个时候可以选择去 [pom.xml](../../pom.xml) 中将`kafka-manager-console`模块注释掉;
+> 注释是因为每次install的时候都会把前端文件`kafka-manager-console`重新打包进`kafka-manager-web`
+> 
+> 完事之后,只需要直接用IDE启动运行`kafka-manager-web`模块中的
+> com.xiaojukeji.kafka.manager.web.MainApplication main方法就行了

kafka-manager-common/pom.xml

@@ -109,10 +109,5 @@
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <scope>compile</scope>
-        </dependency>
     </dependencies>
 </project>

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/Constant.java

@@ -9,7 +9,7 @@ public class Constant {
 
     public static final Integer MAX_AVG_BYTES_DURATION = 10;
 
-    public static final Integer BATCH_INSERT_SIZE = 50;
+    public static final Integer BATCH_INSERT_SIZE = 30;
 
     public static final Integer DEFAULT_SESSION_TIMEOUT_UNIT_MS = 30000;
 

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/topic/MineTopicSummary.java

@@ -25,6 +25,8 @@ public class MineTopicSummary {
 
     private Integer access;
 
+    private String description;
+
     public Long getLogicalClusterId() {
         return logicalClusterId;
     }
@@ -105,6 +107,14 @@ public class MineTopicSummary {
         this.access = access;
     }
 
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
     @Override
     public String toString() {
         return "MineTopicSummary{" +

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/order/OrderVO.java

@@ -27,8 +27,11 @@ public class OrderVO {
     @ApiModelProperty(value = "工单状态, 0:待审批, 1:通过, 2:拒绝, 3:取消")
     private Integer status;
 
-    @ApiModelProperty(value = "申请/审核时间")
+    @ApiModelProperty(value = "申请时间")
-    private Date gmtTime;
+    private Date gmtCreate;
+
+    @ApiModelProperty(value = "审核时间")
+    private Date gmtHandle;
 
     public Long getId() {
         return id;
@@ -70,12 +73,20 @@ public class OrderVO {
         this.status = status;
     }
 
-    public Date getGmtTime() {
+    public Date getGmtCreate() {
-        return gmtTime;
+        return gmtCreate;
     }
 
-    public void setGmtTime(Date gmtTime) {
+    public void setGmtCreate(Date gmtCreate) {
-        this.gmtTime = gmtTime;
+        this.gmtCreate = gmtCreate;
+    }
+
+    public Date getGmtHandle() {
+        return gmtHandle;
+    }
+
+    public void setGmtHandle(Date gmtHandle) {
+        this.gmtHandle = gmtHandle;
     }
 
     public String getApplicant() {
@@ -95,7 +106,7 @@ public class OrderVO {
                 ", applicant='" + applicant + '\'' +
                 ", description='" + description + '\'' +
                 ", status=" + status +
-                ", gmtTime=" + gmtTime +
+                ", gmtTime=" + gmtCreate +
                 '}';
     }
 }

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/topic/TopicMineVO.java

@@ -36,6 +36,9 @@ public class TopicMineVO {
     @ApiModelProperty(value = "状态, 0:无权限, 1:可消费 2:可发送 3:可消费发送 4:可管理")
     private Integer access;
 
+    @ApiModelProperty(value = "备注")
+    private String description;
+
     public Long getClusterId() {
         return clusterId;
     }
@@ -108,6 +111,14 @@ public class TopicMineVO {
         this.access = access;
     }
 
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
     @Override
     public String toString() {
         return "TopicMineVO{" +

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/events/RegionCreatedEvent.java

@@ -0,0 +1,20 @@
+package com.xiaojukeji.kafka.manager.common.events;
+
+import com.xiaojukeji.kafka.manager.common.entity.pojo.RegionDO;
+import lombok.Getter;
+import org.springframework.context.ApplicationEvent;
+
+/**
+ * Region创建事件
+ * @author zengqiao
+ * @date 22/01/1
+ */
+@Getter
+public class RegionCreatedEvent extends ApplicationEvent {
+    private final RegionDO regionDO;
+
+    public RegionCreatedEvent(Object source, RegionDO regionDO) {
+        super(source);
+        this.regionDO = regionDO;
+    }
+}

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/BackoffUtils.java

@@ -0,0 +1,20 @@
+package com.xiaojukeji.kafka.manager.common.utils;
+
+public class BackoffUtils {
+    private BackoffUtils() {
+    }
+
+    public static void backoff(long timeUnitMs) {
+        if (timeUnitMs <= 0) {
+            return;
+        }
+
+        try {
+            Thread.sleep(timeUnitMs);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        } catch (Exception e) {
+            // ignore
+        }
+    }
+}

kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java

@@ -1,5 +1,6 @@
 package com.xiaojukeji.kafka.manager.common.utils.jmx;
 
+import com.xiaojukeji.kafka.manager.common.utils.BackoffUtils;
 import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,7 +14,6 @@ import javax.naming.Context;
 import javax.rmi.ssl.SslRMIClientSocketFactory;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -25,19 +25,25 @@ import java.util.concurrent.atomic.AtomicInteger;
  * @date 2015/11/9.
  */
 public class JmxConnectorWrap {
-    private final static Logger LOGGER = LoggerFactory.getLogger(JmxConnectorWrap.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(JmxConnectorWrap.class);
 
-    private String host;
+    private final Long physicalClusterId;
 
-    private int port;
+    private final Integer brokerId;
+
+    private final String host;
+
+    private final int port;
 
     private JMXConnector jmxConnector;
 
-    private AtomicInteger atomicInteger;
+    private final AtomicInteger atomicInteger;
 
     private JmxConfig jmxConfig;
 
-    public JmxConnectorWrap(String host, int port, JmxConfig jmxConfig) {
+    public JmxConnectorWrap(Long physicalClusterId, Integer brokerId, String host, int port, JmxConfig jmxConfig) {
+        this.physicalClusterId = physicalClusterId;
+        this.brokerId = brokerId;
         this.host = host;
         this.port = port;
         this.jmxConfig = jmxConfig;
@@ -67,7 +73,7 @@ public class JmxConnectorWrap {
         try {
             jmxConnector.close();
         } catch (IOException e) {
-            LOGGER.warn("close JmxConnector exception, host:{} port:{}.", host, port, e);
+            LOGGER.warn("close JmxConnector exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
         }
     }
 
@@ -90,12 +96,12 @@ public class JmxConnectorWrap {
             }
 
             jmxConnector = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), environment);
-            LOGGER.info("JMX connect success, host:{} port:{}.", host, port);
+            LOGGER.info("JMX connect success, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port);
             return true;
         } catch (MalformedURLException e) {
-            LOGGER.error("JMX url exception, host:{} port:{} jmxUrl:{}", host, port, jmxUrl, e);
+            LOGGER.error("JMX url exception, physicalClusterId:{} brokerId:{} host:{} port:{} jmxUrl:{}", physicalClusterId, brokerId, host, port, jmxUrl, e);
         } catch (Exception e) {
-            LOGGER.error("JMX connect exception, host:{} port:{}.", host, port, e);
+            LOGGER.error("JMX connect exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
         }
         return false;
     }
@@ -146,18 +152,16 @@ public class JmxConnectorWrap {
         long now = System.currentTimeMillis();
         while (true) {
             try {
-                if (System.currentTimeMillis() - now > 60000) {
-                    break;
-                }
                 int num = atomicInteger.get();
                 if (num <= 0) {
-                    Thread.sleep(2);
+                    BackoffUtils.backoff(2);
-                    continue;
                 }
-                if (atomicInteger.compareAndSet(num, num - 1)) {
+
+                if (atomicInteger.compareAndSet(num, num - 1) || System.currentTimeMillis() - now > 6000) {
                     break;
                 }
             } catch (Exception e) {
+                // ignore
             }
         }
     }

kafka-manager-console/package.json

@@ -1,9 +1,10 @@
 {
   "name": "logi-kafka",
-  "version": "2.4.3",
+  "version": "2.5.0",
   "description": "",
   "scripts": {
-    "start": "webpack-dev-server",
+    "prestart": "npm install --save-dev webpack-dev-server",
+    "start": "webpack serve",
     "daily-build": "cross-env NODE_ENV=production webpack",
     "pre-build": "cross-env NODE_ENV=production webpack",
     "prod-build": "cross-env NODE_ENV=production webpack",
@@ -13,18 +14,19 @@
   "license": "ISC",
   "devDependencies": {
     "@hot-loader/react-dom": "^16.8.6",
-    "@types/echarts": "^4.4.1",
+    "@types/events": "^3.0.0",
     "@types/lodash.debounce": "^4.0.6",
     "@types/react": "^16.8.8",
     "@types/react-dom": "^16.8.2",
     "@types/react-router-dom": "^4.3.1",
     "@types/spark-md5": "^3.0.2",
+    "@webpack-cli/serve": "^1.6.0",
     "antd": "^3.26.15",
     "clean-webpack-plugin": "^3.0.0",
     "clipboard": "^2.0.8",
     "cross-env": "^7.0.2",
     "css-loader": "^2.1.0",
-    "echarts": "^4.5.0",
+    "echarts": "^5.2.1",
     "file-loader": "^5.0.2",
     "html-webpack-plugin": "^3.2.0",
     "increase-memory-limit": "^1.0.7",
@@ -50,11 +52,10 @@
     "typescript": "^3.3.3333",
     "url-loader": "^4.1.1",
     "webpack": "^4.29.6",
-    "webpack-cli": "^3.2.3",
+    "webpack-cli": "^4.9.1",
-    "webpack-dev-server": "^3.2.1",
     "xlsx": "^0.16.1"
   },
   "dependencies": {
     "format-to-json": "^1.0.4"
   }
-}
+}

kafka-manager-console/src/component/chart/bar-chart.tsx

@@ -1,14 +1,29 @@
 import * as React from 'react';
 import { Spin, notification } from 'component/antd';
-import echarts, { EChartOption } from 'echarts/lib/echarts';
+import * as echarts from 'echarts/core';
 
 // 引入柱状图
-import 'echarts/lib/chart/bar';
+import { BarChart } from 'echarts/charts';
 
 // 引入提示框和标题组件
-import 'echarts/lib/component/tooltip';
+import {
-import 'echarts/lib/component/title';
+  TitleComponent,
-import 'echarts/lib/component/legend';
+  TooltipComponent,
+  LegendComponent,
+  GridComponent,
+} from 'echarts/components';
+import { CanvasRenderer } from 'echarts/renderers';
+import { EChartsOption } from 'echarts';
+
+// 注册必须的组件
+echarts.use([
+  TitleComponent,
+  LegendComponent,
+  TooltipComponent,
+  BarChart,
+  GridComponent,
+  CanvasRenderer,
+]);
 
 interface IChartProps {
   getChartData: any;
@@ -38,7 +53,7 @@ export class BarChartComponet extends React.Component<IChartProps> {
     this.chart.resize();
   }
 
-  public isHasData = (data: EChartOption) => {
+  public isHasData = (data: any) => {
     const noData = !(data.series && data.series.length);
     this.setState({ noData });
     return !noData;
@@ -54,7 +69,7 @@ export class BarChartComponet extends React.Component<IChartProps> {
     const chartOptions = getChartData();
 
     if ((typeof chartOptions.then) === 'function') {
-      return chartOptions.then((data: EChartOption) => {
+      return chartOptions.then((data: EChartsOption) => {
         this.setState({ loading: false });
 
         if (this.isHasData(data)) {

kafka-manager-console/src/component/chart/date-picker-chart.tsx

@@ -3,16 +3,34 @@ import { DatePicker, notification, Spin } from 'component/antd';
 import moment, { Moment } from 'moment';
 import { timeStampStr } from 'constants/strategy';
 import { disabledDate } from 'lib/utils';
-import echarts from 'echarts';
+import * as echarts from 'echarts/core';
 
-// 引入柱状图和折线图
+// 引入柱状图
-import 'echarts/lib/chart/bar';
+import { BarChart, LineChart } from 'echarts/charts';
-import 'echarts/lib/chart/line';
 
 // 引入提示框和标题组件
-import 'echarts/lib/component/tooltip';
+import {
-import 'echarts/lib/component/title';
+  TitleComponent,
-import 'echarts/lib/component/legend';
+  TooltipComponent,
+  LegendComponent,
+  GridComponent,
+  MarkLineComponent,
+  DatasetComponent,
+} from 'echarts/components';
+import { CanvasRenderer } from 'echarts/renderers';
+
+// 注册必须的组件
+echarts.use([
+  TitleComponent,
+  LegendComponent,
+  TooltipComponent,
+  GridComponent,
+  BarChart,
+  LineChart,
+  CanvasRenderer,
+  DatasetComponent,
+  MarkLineComponent,
+]);
 import './index.less';
 
 const { RangePicker } = DatePicker;
@@ -60,6 +78,23 @@ export class ChartWithDatePicker extends React.Component<IChartProps> {
   public changeChartOptions(options: any) {
     const noData = options.series.length ? false : true;
     this.setState({ noData });
+    options.tooltip.formatter = (params: any) => {
+      let res =
+        '<div style=\'margin-bottom:5px;padding:0 12px;width:100%;height:24px;line-height:24px;border-radius:3px;\'><p>' +
+        params[0].data.time +
+        ' </p></div>';
+      // tslint:disable-next-line:prefer-for-of
+      for (let i = 0; i < params.length; i++) {
+        res += `<div key=${params[i].seriesName} style="color: #fff;padding:0 12px;line-height: 24px">
+                  <span style="display:inline-block;margin-right:5px;border-radius:50%;width:10px;height:10px;background-color:${[
+            params[i].color,
+          ]};"></span>
+                  ${params[i].seriesName}
+                  ${params[i].data[params[i].seriesName]}
+                </div>`;
+      }
+      return res;
+    };
     this.chart.setOption(options, true);
   }
 
@@ -79,7 +114,7 @@ export class ChartWithDatePicker extends React.Component<IChartProps> {
   public render() {
     const { customerNode } = this.props;
     return (
-      <div className="status-box" style={{minWidth: '930px'}}>
+      <div className="status-box" style={{ minWidth: '930px' }}>
         <div className="status-graph">
           <div className="k-toolbar">
             {customerNode}

kafka-manager-console/src/component/chart/doughnut-chart.tsx

@@ -1,13 +1,27 @@
 import * as React from 'react';
 import { Spin } from 'component/antd';
-import echarts from 'echarts/lib/echarts';
+import * as echarts from 'echarts/core';
-// 引入饼状图
+// 引入饼图
-import 'echarts/lib/chart/pie';
+import { PieChart } from 'echarts/charts';
-// 引入提示框和标题组件
-import 'echarts/lib/component/tooltip';
-import 'echarts/lib/component/title';
-import 'echarts/lib/component/legend';
 
+// 引入提示框和标题组件
+import {
+  TitleComponent,
+  TooltipComponent,
+  LegendComponent,
+  GridComponent,
+} from 'echarts/components';
+import { CanvasRenderer } from 'echarts/renderers';
+
+// 注册必须的组件
+echarts.use([
+  PieChart,
+  TitleComponent,
+  LegendComponent,
+  TooltipComponent,
+  GridComponent,
+  CanvasRenderer,
+]);
 interface IPieProps {
   getChartData: any;
 }

kafka-manager-console/src/component/chart/line-chart.tsx

@@ -1,25 +1,45 @@
 import React from 'react';
-import echarts, { EChartOption } from 'echarts/lib/echarts';
+import * as echarts from 'echarts/core';
-import 'echarts/lib/chart/pie';
-import 'echarts/lib/chart/line';
-import 'echarts/lib/component/legend';
-import 'echarts/lib/component/tooltip';
-import 'echarts/lib/component/title';
-import 'echarts/lib/component/axis';
 import './index.less';
 
+// 引入柱状图
+import { PieChart, LineChart } from 'echarts/charts';
+
+// 引入提示框和标题组件
+import {
+  TitleComponent,
+  TooltipComponent,
+  LegendComponent,
+  GridComponent,
+  ToolboxComponent,
+  DatasetComponent,
+} from 'echarts/components';
+import { CanvasRenderer } from 'echarts/renderers';
+
+// 注册必须的组件
+echarts.use([
+  PieChart,
+  LineChart,
+  ToolboxComponent,
+  TitleComponent,
+  LegendComponent,
+  TooltipComponent,
+  GridComponent,
+  DatasetComponent,
+  CanvasRenderer,
+]);
 export interface IEchartsProps {
   width?: number;
   height?: number;
-  options?: EChartOption;
+  options?: any;
 }
 
-export const hasData = (options: EChartOption) => {
+export const hasData = (options: any) => {
   if (options && options.series && options.series.length) return true;
   return false;
 };
 
-export default class LineChart extends React.Component<IEchartsProps> {
+export default class LineCharts extends React.Component<IEchartsProps> {
   public id = null as HTMLDivElement;
 
   public myChart = null as echarts.ECharts;
@@ -27,7 +47,7 @@ export default class LineChart extends React.Component<IEchartsProps> {
   public componentDidMount() {
     const { options } = this.props;
     this.myChart = echarts.init(this.id);
-    this.myChart.setOption(options);
+    this.myChart.setOption(options, true);
     window.addEventListener('resize', this.resize);
   }
 
@@ -41,7 +61,7 @@ export default class LineChart extends React.Component<IEchartsProps> {
 
   public refresh = () => {
     const { options } = this.props;
-    this.myChart.setOption(options);
+    this.myChart.setOption(options, true);
   }
 
   public resize = () => {
@@ -50,6 +70,6 @@ export default class LineChart extends React.Component<IEchartsProps> {
 
   public render() {
     const { height, width } = this.props;
-    return <div ref={id => this.id = id} style={{width: `${width}px`, height: `${height}px`}} />;
+    return <div ref={id => this.id = id} style={{ width: `${width}px`, height: `${height}px` }} />;
   }
 }

kafka-manager-console/src/container/admin/data-curve/config.ts

@@ -1,4 +1,3 @@
-import { EChartOption } from 'echarts/lib/echarts';
 import moment from 'moment';
 import { ICurve } from 'container/common-curve/config';
 import { adminMonitor } from 'store/admin-monitor';
@@ -124,7 +123,7 @@ export interface ICurveType {
   type: curveType;
   title: string;
   curves: ICurve[];
-  parser: (option: ICurve, data: any[]) => EChartOption;
+  parser: (option: ICurve, data: any[]) => any;
 }
 
 export const byteTypeCurves: ICurveType[] = [

kafka-manager-console/src/container/admin/data-curve/parser.ts

@@ -1,5 +1,5 @@
 import moment from 'moment';
-import { EChartOption } from 'echarts';
+import { EChartsOption } from 'echarts';
 import { ICurve, ILineData, baseLineLegend, baseLineGrid, baseAxisStyle, noAxis, UNIT_HEIGHT } from 'container/common-curve/config';
 import { IClusterMetrics, ISeriesOption } from 'types/base-type';
 import { timeFormat } from 'constants/strategy';
@@ -48,20 +48,20 @@ export const getBaseOptions = (option: ICurve, data: ILineData[]) => {
         return Number(i.value);
       }),
     }],
-  } as EChartOption;
+  } as EChartsOption;
 };
 
-export const parseLine = (option: ICurve, data: ILineData[]): EChartOption => {
+export const parseLine = (option: ICurve, data: ILineData[]): EChartsOption => {
   return Object.assign({}, getBaseOptions(option, data), {
     legend: {
       ...baseLineLegend,
       bottom: '0',
       align: 'auto',
     },
-  }) as EChartOption;
+  }) as EChartsOption;
 };
 
-export const parseBrokerMetricOption = (option: ICurve, data: IClusterMetrics[]): EChartOption => {
+export const parseBrokerMetricOption = (option: ICurve, data: IClusterMetrics[]): EChartsOption => {
   let name;
   let series: ISeriesOption[];
   data = data || [];

kafka-manager-console/src/container/alarm/alarm-detail/history-detail.tsx

@@ -6,7 +6,7 @@ import { alarm } from 'store/alarm';
 import { observer } from 'mobx-react';
 import { handlePageBack } from 'lib/utils';
 import LineChart, { hasData } from 'component/chart/line-chart';
-import { EChartOption } from 'echarts';
+import { EChartsOption } from 'echarts';
 import { timeFormat } from 'constants/strategy';
 import Url from 'lib/url-parser';
 import moment = require('moment');
@@ -40,7 +40,7 @@ export class HistoryDetail extends React.Component {
     return <div className="no-data-info" style={{ ...style }} key="loading"><Spin /></div>;
   }
 
-  public renderEchart = (options: EChartOption, loading = false) => {
+  public renderEchart = (options: EChartsOption, loading = false) => {
     const data = hasData(options);
     if (loading) return this.renderLoading(400);
     if (!data) return this.renderNoData(400);
@@ -51,7 +51,7 @@ export class HistoryDetail extends React.Component {
   }
 
   public renderHistoricalTraffic(metric: IMonitorMetric) {
-    const option = this.getChartOption() as EChartOption;
+    const option = this.getChartOption() as EChartsOption;
 
     return (
       <>

kafka-manager-console/src/container/common-curve/config.ts

@@ -1,5 +1,4 @@
-import { EChartOption } from 'echarts/lib/echarts';
+import { EChartsOption } from 'echarts';
-import moment from 'moment';
 
 export interface ILineData {
   value: number;
@@ -9,7 +8,7 @@ export interface ICurve {
   title?: string;
   path: string;
   colors: string[];
-  parser?: (option: ICurve, data: ILineData) => EChartOption;
+  parser?: (option: ICurve, data: ILineData) => EChartsOption;
   message?: string;
   unit?: string;
   api?: any;
@@ -69,13 +68,13 @@ export const noAxis = {
   },
 };
 
-export const getHight = (options: EChartOption) => {
+export const getHight = (options: any) => {
-  let grid = options ? options.grid as EChartOption.Grid : null;
+  let grid = options ? options.grid : null;
   if (!options || !grid) grid = baseLineGrid;
   return Number(grid.height) + getLegendHight(options) + Number(grid.top) + LEGEND_PADDING + UNIT_HEIGHT;
 };
 
-export const getLegendHight = (options: EChartOption) => {
+export const getLegendHight = (options: any) => {
   if (!options) return 0;
   if (options.legend.show === false) return 0;
   const legendHight = options.legend.textStyle.lineHeight + defaultLegendPadding;

kafka-manager-console/src/container/common-curve/index.tsx

@@ -1,4 +1,4 @@
-import { EChartOption } from 'echarts';
+import { EChartsOption } from 'echarts';
 import { observer } from 'mobx-react';
 import React from 'react';
 import { curveInfo } from 'store/curve-info';
@@ -10,7 +10,7 @@ import LineChart, { hasData } from 'component/chart/line-chart';
 
 export interface ICommonCurveProps {
   options: ICurve;
-  parser?: (option: ICurve, data: any[]) => EChartOption;
+  parser?: (option: ICurve, data: any[]) => any;
 }
 
 @observer
@@ -41,7 +41,7 @@ export class CommonCurve extends React.Component<ICommonCurveProps> {
     fullScreen.show(this.renderCurve(options, loading, true));
   }
 
-  public renderOpBtns = (options: EChartOption, expand = false) => {
+  public renderOpBtns = (options: EChartsOption, expand = false) => {
     const data = hasData(options);
     return (
       <div className="charts-op" key="op">
@@ -85,7 +85,7 @@ export class CommonCurve extends React.Component<ICommonCurveProps> {
     return <div className="no-data-info" style={{ ...style }} key="loading"><Spin /></div>;
   }
 
-  public renderEchart = (options: EChartOption, loading = false) => {
+  public renderEchart = (options: EChartsOption, loading = false) => {
     const height = getHight(options);
     const data = hasData(options);
 
@@ -94,7 +94,7 @@ export class CommonCurve extends React.Component<ICommonCurveProps> {
     return <LineChart height={height} options={options} key="chart" />;
   }
 
-  public renderCurve = (options: EChartOption, loading: boolean, expand = false) => {
+  public renderCurve = (options: any, loading: boolean, expand = false) => {
     const data = hasData(options);
     return (
       <div className="common-chart-wrapper" >

kafka-manager-console/src/container/header/index.tsx

@@ -7,7 +7,7 @@ import { urlPrefix } from 'constants/left-menu';
 import { region, IRegionIdcs } from 'store/region';
 import logoUrl from '../../assets/image/kafka-logo.png';
 import userIcon from '../../assets/image/normal.png';
-import weChat from '../../assets/image/wechat.jpeg';
+import weChat from '../../assets/image/wechat.png';
 import { users } from 'store/users';
 import { observer } from 'mobx-react';
 import { Link } from 'react-router-dom';
@@ -60,8 +60,8 @@ export const Header = observer((props: IHeader) => {
     });
   };
   const content = (
-    <div style={{ height: '250px', padding: '5px' }} className="kafka-avatar-img">
+    <div style={{ height: '200px', padding: '5px' }} className="kafka-avatar-img">
-      <img style={{ width: '190px', height: '246px' }} src={weChat} alt="" />
+      <img style={{ width: '190px', height: '190px' }} src={weChat} alt="" />
     </div>
   );
   const helpCenter = (
@@ -144,8 +144,8 @@ export const Header = observer((props: IHeader) => {
     <div className="kafka-header-container">
       <div className="left-content">
         <img className="kafka-header-icon" src={logoUrl} alt="" />
-        <span className="kafka-header-text">Kafka Manager</span>
+        <span className="kafka-header-text">LogiKM</span>
-        <a className='kafka-header-version' href="https://github.com/didi/Logi-KafkaManager/releases" target='_blank'>v2.4.2</a>
+        <a className='kafka-header-version' href="https://github.com/didi/Logi-KafkaManager/releases" target='_blank'>v2.5.0</a>
         {/* 添加版本超链接 */}
       </div>
       <div className="mid-content">

kafka-manager-console/src/container/user-center/order-list.tsx

@@ -115,11 +115,19 @@ export class OrderList extends SearchAndFilterContainer {
       status,
       {
         title: '申请时间',
-        dataIndex: 'gmtTime',
+        dataIndex: 'gmtCreate',
-        key: 'gmtTime',
+        key: 'gmtCreate',
-        sorter: (a: IBaseOrder, b: IBaseOrder) => b.gmtTime - a.gmtTime,
+        sorter: (a: IBaseOrder, b: IBaseOrder) => b.gmtCreate - a.gmtCreate,
-        render: (t: number) => moment(t).format(timeFormat),
+        render: (t: number) => t ? moment(t).format(timeFormat) : '-',
-      }, {
+      },
+      {
+        title: '审批时间',
+        dataIndex: 'gmtHandle',
+        key: 'gmtHandle',
+        sorter: (a: IBaseOrder, b: IBaseOrder) => b.gmtHandle - a.gmtHandle,
+        render: (t: number) => t ? moment(t).format(timeFormat) : '-',
+      },
+      {
         title: '操作',
         key: 'operation',
         dataIndex: 'operation',

kafka-manager-console/src/routers/index.htm

@@ -1,12 +1,15 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=2">
-  <title>KafkaManager</title>
+  <title>LogiKM</title>
 </head>
+
 <body>
   <div id="root"></div>
   <div id="modal"></div>
 </body>
+
 </html>

kafka-manager-console/src/store/curve-info.ts

@@ -1,6 +1,6 @@
 import { observable, action } from 'mobx';
 import moment = require('moment');
-import { EChartOption } from 'echarts/lib/echarts';
+import { EChartsOption } from 'echarts';
 import { ICurve } from 'container/common-curve/config';
 import { curveKeys, PERIOD_RADIO_MAP } from 'container/admin/data-curve/config';
 import { timeFormat } from 'constants/strategy';
@@ -13,7 +13,7 @@ class CurveInfo {
   public timeRange: [moment.Moment, moment.Moment] = PERIOD_RADIO_MAP.get(this.periodKey).dateRange;
 
   @observable
-  public curveData: { [key: string]: EChartOption } = {};
+  public curveData: { [key: string]: EChartsOption } = {};
 
   @observable
   public curveLoading: { [key: string]: boolean } = {};
@@ -25,7 +25,7 @@ class CurveInfo {
   public currentOperator: string;
 
   @action.bound
-  public setCurveData(key: curveKeys | string, data: EChartOption) {
+  public setCurveData(key: curveKeys | string, data: EChartsOption) {
     this.curveData[key] = data;
   }
 
@@ -59,7 +59,7 @@ class CurveInfo {
 
   public getCommonCurveData = (
     options: ICurve,
-    parser: (option: ICurve, data: any[]) => EChartOption,
+    parser: (option: ICurve, data: any[]) => EChartsOption,
     reload?: boolean) => {
     const { path } = options;
     this.setCurveData(path, null);

kafka-manager-console/webpack.config.js

@@ -122,11 +122,11 @@ module.exports = {
     },
   },
   devServer: {
-    contentBase: outPath,
+    // contentBase: outPath,
     host: '127.0.0.1',
     port: 1025,
     hot: true,
-    disableHostCheck: true,
+    // disableHostCheck: true,
     historyApiFallback: true,
     proxy: {
       '/api/v1/': {

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/LogicalClusterMetadataManager.java

@@ -208,7 +208,8 @@ public class LogicalClusterMetadataManager {
         // 计算逻辑集群到Topic名称的映射
         Set<String> topicNameSet = PhysicalClusterMetadataManager.getBrokerTopicNum(
                 logicalClusterDO.getClusterId(),
-                brokerIdSet);
+                brokerIdSet
+        );
         LOGICAL_CLUSTER_ID_TOPIC_NAME_MAP.put(logicalClusterDO.getId(), topicNameSet);
 
         // 计算Topic名称到逻辑集群的映射

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java

@@ -317,7 +317,7 @@ public class PhysicalClusterMetadataManager {
         metadataMap.put(brokerId, brokerMetadata);
 
         Map<Integer, JmxConnectorWrap> jmxMap = JMX_CONNECTOR_MAP.getOrDefault(clusterId, new ConcurrentHashMap<>());
-        jmxMap.put(brokerId, new JmxConnectorWrap(brokerMetadata.getHost(), brokerMetadata.getJmxPort(), jmxConfig));
+        jmxMap.put(brokerId, new JmxConnectorWrap(clusterId, brokerId, brokerMetadata.getHost(), brokerMetadata.getJmxPort(), jmxConfig));
         JMX_CONNECTOR_MAP.put(clusterId, jmxMap);
 
         Map<Integer, KafkaVersion> versionMap = KAFKA_VERSION_MAP.getOrDefault(clusterId, new ConcurrentHashMap<>());
@@ -542,9 +542,12 @@ public class PhysicalClusterMetadataManager {
     }
 
     public static Set<String> getBrokerTopicNum(Long clusterId, Set<Integer> brokerIdSet) {
-        Set<String> topicNameSet = new HashSet<>();
-
         Map<String, TopicMetadata> metadataMap = TOPIC_METADATA_MAP.get(clusterId);
+        if (metadataMap == null) {
+            return new HashSet<>();
+        }
+
+        Set<String> topicNameSet = new HashSet<>();
         for (String topicName: metadataMap.keySet()) {
             try {
                 TopicMetadata tm = metadataMap.get(topicName);

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicExpiredService.java

@@ -13,4 +13,12 @@ public interface TopicExpiredService {
     List<TopicExpiredData> getExpiredTopicDataList(String username);
 
     ResultStatus retainExpiredTopic(Long physicalClusterId, String topicName, Integer retainDays);
+
+    /**
+     * 通过topictopic名称删除
+     * @param clusterId 集群id
+     * @param topicName topic名称
+     * @return int
+     */
+    int deleteByTopicName(Long clusterId, String topicName);
 }

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java

@@ -185,7 +185,8 @@ public class GatewayConfigServiceImpl implements GatewayConfigService {
             List<GatewayConfigDO> gatewayConfigDOList = gatewayConfigDao.getByConfigType(gatewayConfigDO.getType());
             Long version = 1L;
             for (GatewayConfigDO elem: gatewayConfigDOList) {
-                if (elem.getVersion() > version) {
+                if (elem.getVersion() >= version) {
+                    // 大于等于的情况下，都需要+1
                     version = elem.getVersion() + 1L;
                 }
             }
@@ -204,6 +205,7 @@ public class GatewayConfigServiceImpl implements GatewayConfigService {
     @Override
     public Result deleteById(Long id) {
         try {
+            // TODO 删除的时候，不能直接删，也需要变更一下version
             if (gatewayConfigDao.deleteById(id) > 0) {
                 return Result.buildSuc();
             }
@@ -232,7 +234,8 @@ public class GatewayConfigServiceImpl implements GatewayConfigService {
             List<GatewayConfigDO> gatewayConfigDOList = gatewayConfigDao.getByConfigType(newGatewayConfigDO.getType());
             Long version = 1L;
             for (GatewayConfigDO elem: gatewayConfigDOList) {
-                if (elem.getVersion() > version) {
+                if (elem.getVersion() >= version) {
+                    // 大于等于的情况下，都需要+1
                     version = elem.getVersion() + 1L;
                 }
             }

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java

@@ -43,6 +43,9 @@ public class AdminServiceImpl implements AdminService {
     @Autowired
     private TopicManagerService topicManagerService;
 
+    @Autowired
+    private TopicExpiredService topicExpiredService;
+
     @Autowired
     private TopicService topicService;
 
@@ -143,6 +146,7 @@ public class AdminServiceImpl implements AdminService {
 
         // 3. 数据库中删除topic
         topicManagerService.deleteByTopicName(clusterDO.getId(), topicName);
+        topicExpiredService.deleteByTopicName(clusterDO.getId(), topicName);
 
         // 4. 数据库中删除authority
         authorityService.deleteAuthorityByTopic(clusterDO.getId(), topicName);

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java

@@ -19,6 +19,8 @@ import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager;
 import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
 import com.xiaojukeji.kafka.manager.service.service.*;
 import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -210,7 +212,7 @@ public class ClusterServiceImpl implements ClusterService {
 
         ZooKeeper zk = null;
         try {
-            zk = new ZooKeeper(zookeeper, 1000, null);
+            zk = new ZooKeeper(zookeeper, 1000, watchedEvent -> LOGGER.info(" receive event : " + watchedEvent.getType().name()));
             for (int i = 0; i < 15; ++i) {
                 if (zk.getState().isConnected()) {
                     // 只有状态是connected的时候，才表示地址是合法的

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/RegionServiceImpl.java

@@ -2,6 +2,8 @@ package com.xiaojukeji.kafka.manager.service.service.impl;
 
 import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
 import com.xiaojukeji.kafka.manager.common.entity.pojo.RegionDO;
+import com.xiaojukeji.kafka.manager.common.events.RegionCreatedEvent;
+import com.xiaojukeji.kafka.manager.common.utils.SpringTool;
 import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
 import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata;
 import com.xiaojukeji.kafka.manager.dao.RegionDao;
@@ -59,6 +61,8 @@ public class RegionServiceImpl implements RegionService {
                 return ResultStatus.BROKER_NOT_EXIST;
             }
             if (regionDao.insert(regionDO) > 0) {
+                // 发布region创建事件
+                SpringTool.publish(new RegionCreatedEvent(this, regionDO));
                 return ResultStatus.SUCCESS;
             }
         } catch (DuplicateKeyException e) {

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicExpiredServiceImpl.java

@@ -75,4 +75,14 @@ public class TopicExpiredServiceImpl implements TopicExpiredService {
         }
         return ResultStatus.MYSQL_ERROR;
     }
+
+    @Override
+    public int deleteByTopicName(Long clusterId, String topicName) {
+        try {
+            return topicExpiredDao.deleteByName(clusterId, topicName);
+        } catch (Exception e) {
+            LOGGER.error("delete topic failed, clusterId:{} topicName:{}", clusterId, topicName, e);
+        }
+        return 0;
+    }
 }

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java

@@ -210,7 +210,7 @@ public class TopicManagerServiceImpl implements TopicManagerService {
             }
         }
 
-        // 增加流量信息
+        // 增加流量和描述信息
         Map<Long, Map<String, TopicMetrics>> metricMap = KafkaMetricsCache.getAllTopicMetricsFromCache();
         for (MineTopicSummary mineTopicSummary : summaryList) {
             TopicMetrics topicMetrics = getTopicMetricsFromCacheOrJmx(
@@ -219,6 +219,10 @@ public class TopicManagerServiceImpl implements TopicManagerService {
                     metricMap);
             mineTopicSummary.setBytesIn(topicMetrics.getSpecifiedMetrics("BytesInPerSecOneMinuteRate"));
             mineTopicSummary.setBytesOut(topicMetrics.getSpecifiedMetrics("BytesOutPerSecOneMinuteRate"));
+
+            // 增加topic描述信息
+            TopicDO topicDO = topicDao.getByTopicName(mineTopicSummary.getPhysicalClusterId(), mineTopicSummary.getTopicName());
+            mineTopicSummary.setDescription(topicDO.getDescription());
         }
         return summaryList;
     }

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/utils/ConfigUtils.java

@@ -1,5 +1,6 @@
 package com.xiaojukeji.kafka.manager.service.utils;
 
+import lombok.Data;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
@@ -8,38 +9,15 @@ import org.springframework.stereotype.Service;
  * @author zengqiao
  * @date 20/4/26
  */
+@Data
 @Service("configUtils")
 public class ConfigUtils {
-    @Value(value = "${custom.idc}")
+    private ConfigUtils() {
+    }
+
+    @Value(value = "${custom.idc:cn}")
     private String idc;
 
     @Value(value = "${spring.profiles.active}")
     private String kafkaManagerEnv;
-
-    @Value(value = "${custom.store-metrics-task.save-days}")
-    private Long maxMetricsSaveDays;
-
-    public String getIdc() {
-        return idc;
-    }
-
-    public void setIdc(String idc) {
-        this.idc = idc;
-    }
-
-    public String getKafkaManagerEnv() {
-        return kafkaManagerEnv;
-    }
-
-    public void setKafkaManagerEnv(String kafkaManagerEnv) {
-        this.kafkaManagerEnv = kafkaManagerEnv;
-    }
-
-    public Long getMaxMetricsSaveDays() {
-        return maxMetricsSaveDays;
-    }
-
-    public void setMaxMetricsSaveDays(Long maxMetricsSaveDays) {
-        this.maxMetricsSaveDays = maxMetricsSaveDays;
-    }
 }

kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/ControllerStateListener.java

@@ -19,13 +19,13 @@ import org.springframework.dao.DuplicateKeyException;
  * @date 20/5/14
  */
 public class ControllerStateListener implements StateChangeListener {
-    private final static Logger LOGGER = LoggerFactory.getLogger(ControllerStateListener.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ControllerStateListener.class);
 
-    private Long clusterId;
+    private final Long clusterId;
 
-    private ZkConfigImpl zkConfig;
+    private final ZkConfigImpl zkConfig;
 
-    private ControllerDao controllerDao;
+    private final ControllerDao controllerDao;
 
     public ControllerStateListener(Long clusterId, ZkConfigImpl zkConfig, ControllerDao controllerDao) {
         this.clusterId = clusterId;
@@ -35,8 +35,11 @@ public class ControllerStateListener implements StateChangeListener {
 
     @Override
     public void init() {
+        if (!checkNodeExist()) {
+            LOGGER.warn("kafka-controller data not exist, clusterId:{}.", clusterId);
+            return;
+        }
         processControllerChange();
-        return;
     }
 
     @Override
@@ -49,12 +52,21 @@ public class ControllerStateListener implements StateChangeListener {
                     break;
             }
         } catch (Exception e) {
-            LOGGER.error("process controller state change failed, clusterId:{} state:{} path:{}.",
+            LOGGER.error("process controller state change failed, clusterId:{} state:{} path:{}.", clusterId, state, path, e);
-                    clusterId, state, path, e);
         }
     }
 
-    private void processControllerChange(){
+    private boolean checkNodeExist() {
+        try {
+            return zkConfig.checkPathExists(ZkPathUtil.CONTROLLER_ROOT_NODE);
+        } catch (Exception e) {
+            LOGGER.error("init kafka-controller data failed, clusterId:{}.", clusterId, e);
+        }
+
+        return false;
+    }
+
+    private void processControllerChange() {
         LOGGER.warn("init controllerData or controller change, clusterId:{}.", clusterId);
         ControllerData controllerData = null;
         try {

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/BrokerMetricsDao.java

@@ -20,5 +20,5 @@ public interface BrokerMetricsDao {
      */
     List<BrokerMetricsDO> getBrokerMetrics(Long clusterId, Integer brokerId, Date startTime, Date endTime);
 
-    int deleteBeforeTime(Date endTime);
+    int deleteBeforeTime(Date endTime, Integer limitSize);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/ClusterMetricsDao.java

@@ -10,5 +10,5 @@ public interface ClusterMetricsDao {
 
     List<ClusterMetricsDO> getClusterMetrics(long clusterId, Date startTime, Date endTime);
 
-    int deleteBeforeTime(Date endTime);
+    int deleteBeforeTime(Date endTime, Integer limitSize);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicAppMetricsDao.java

@@ -30,5 +30,5 @@ public interface TopicAppMetricsDao {
      * @param endTime
      * @return
      */
-    int deleteBeforeTime(Date endTime);
+    int deleteBeforeTime(Date endTime, Integer limitSize);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicExpiredDao.java

@@ -17,4 +17,6 @@ public interface TopicExpiredDao {
     int replace(TopicExpiredDO expiredDO);
 
     TopicExpiredDO getByTopic(Long clusterId, String topicName);
+
+    int deleteByName(Long clusterId, String topicName);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicMetricsDao.java

@@ -22,5 +22,5 @@ public interface TopicMetricsDao {
 
     List<TopicMetricsDO> getLatestTopicMetrics(Long clusterId, Date afterTime);
 
-    int deleteBeforeTime(Date endTime);
+    int deleteBeforeTime(Date endTime, Integer limitSize);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicRequestMetricsDao.java

@@ -33,9 +33,7 @@ public interface TopicRequestMetricsDao {
      * @param endTime
      * @return
      */
-    int deleteBeforeTime(Date endTime);
+    int deleteBeforeTime(Date endTime, Integer limitSize);
-
-    int deleteBeforeId(Long id);
 
     List<TopicMetricsDO> getById(Long startId, Long endId);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicThrottledMetricsDao.java

@@ -32,5 +32,5 @@ public interface TopicThrottledMetricsDao {
 
     List<TopicThrottledMetricsDO> getLatestTopicThrottledMetrics(Long clusterId, Date afterTime);
 
-    int deleteBeforeTime(Date endTime);
+    int deleteBeforeTime(Date endTime, Integer limitSize);
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/BrokerMetricsImpl.java

@@ -37,7 +37,10 @@ public class BrokerMetricsImpl implements BrokerMetricsDao {
     }
 
     @Override
-    public int deleteBeforeTime(Date endTime) {
+    public int deleteBeforeTime(Date endTime, Integer limitSize) {
-        return sqlSession.delete("BrokerMetricsDao.deleteBeforeTime", endTime);
+        Map<String, Object> params = new HashMap<>(2);
+        params.put("endTime", endTime);
+        params.put("limitSize", limitSize);
+        return sqlSession.delete("BrokerMetricsDao.deleteBeforeTime", params);
     }
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/ClusterMetricsDaoImpl.java

@@ -27,7 +27,7 @@ public class ClusterMetricsDaoImpl implements ClusterMetricsDao {
 
     @Override
     public List<ClusterMetricsDO> getClusterMetrics(long clusterId, Date startTime, Date endTime) {
-        Map<String, Object> map = new HashMap<String, Object>(3);
+        Map<String, Object> map = new HashMap<>(3);
         map.put("clusterId", clusterId);
         map.put("startTime", startTime);
         map.put("endTime", endTime);
@@ -35,7 +35,10 @@ public class ClusterMetricsDaoImpl implements ClusterMetricsDao {
     }
 
     @Override
-    public int deleteBeforeTime(Date endTime) {
+    public int deleteBeforeTime(Date endTime, Integer limitSize) {
-        return sqlSession.delete("ClusterMetricsDao.deleteBeforeTime", endTime);
+        Map<String, Object> params = new HashMap<>(2);
+        params.put("endTime", endTime);
+        params.put("limitSize", limitSize);
+        return sqlSession.delete("ClusterMetricsDao.deleteBeforeTime", params);
     }
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicAppMetricsDaoImpl.java

@@ -46,7 +46,10 @@ public class TopicAppMetricsDaoImpl implements TopicAppMetricsDao {
     }
 
     @Override
-    public int deleteBeforeTime(Date endTime) {
+    public int deleteBeforeTime(Date endTime, Integer limitSize) {
-        return sqlSession.delete("TopicAppMetricsDao.deleteBeforeTime", endTime);
+        Map<String, Object> params = new HashMap<>(2);
+        params.put("endTime", endTime);
+        params.put("limitSize", limitSize);
+        return sqlSession.delete("TopicAppMetricsDao.deleteBeforeTime", params);
     }
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicExpiredDaoImpl.java

@@ -50,4 +50,12 @@ public class TopicExpiredDaoImpl implements TopicExpiredDao {
         params.put("topicName", topicName);
         return sqlSession.selectOne("TopicExpiredDao.getByTopic", params);
     }
+
+    @Override
+    public int deleteByName(Long clusterId, String topicName) {
+        Map<String, Object> params = new HashMap<>(2);
+        params.put("clusterId", clusterId);
+        params.put("topicName", topicName);
+        return sqlSession.delete("TopicExpiredDao.deleteByName", params);
+    }
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicMetricsDaoImpl.java

@@ -60,7 +60,10 @@ public class TopicMetricsDaoImpl implements TopicMetricsDao {
     }
 
     @Override
-    public int deleteBeforeTime(Date endTime) {
+    public int deleteBeforeTime(Date endTime, Integer limitSize) {
-        return sqlSession.delete("TopicMetricsDao.deleteBeforeTime", endTime);
+        Map<String, Object> params = new HashMap<>(2);
+        params.put("endTime", endTime);
+        params.put("limitSize", limitSize);
+        return sqlSession.delete("TopicMetricsDao.deleteBeforeTime", params);
     }
 }

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicRequestMetricsDaoImpl.java

@@ -45,13 +45,11 @@ public class TopicRequestMetricsDaoImpl implements TopicRequestMetricsDao {
     }
 
     @Override
-    public int deleteBeforeTime(Date endTime) {
+    public int deleteBeforeTime(Date endTime, Integer limitSize) {
-        return sqlSession.delete("TopicRequestMetricsDao.deleteBeforeTime", endTime);
+        Map<String, Object> params = new HashMap<>();
-    }
+        params.put("endTime", endTime);
-
+        params.put("limitSize", limitSize);
-    @Override
+        return sqlSession.delete("TopicRequestMetricsDao.deleteBeforeTime", params);
-    public int deleteBeforeId(Long id) {
-        return sqlSession.delete("TopicRequestMetricsDao.deleteBeforeId", id);
     }
 
     @Override

kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicThrottledMetricsDaoImpl.java

@@ -75,7 +75,10 @@ public class TopicThrottledMetricsDaoImpl implements TopicThrottledMetricsDao {
     }
 
     @Override
-    public int deleteBeforeTime(Date endTime) {
+    public int deleteBeforeTime(Date endTime, Integer limitSize) {
-        return sqlSession.delete("TopicThrottledMetricsDao.deleteBeforeTime", endTime);
+        Map<String, Object> params = new HashMap<>(2);
+        params.put("endTime", endTime);
+        params.put("limitSize", limitSize);
+        return sqlSession.delete("TopicThrottledMetricsDao.deleteBeforeTime", params);
     }
 }

kafka-manager-dao/src/main/resources/mapper/BrokerMetricsDao.xml

@@ -29,9 +29,9 @@
 	]]>
 	</select>
 
-	<delete id="deleteBeforeTime" parameterType="java.util.Date">
+	<delete id="deleteBeforeTime" parameterType="java.util.Map">
 		<![CDATA[
-		DELETE FROM broker_metrics WHERE gmt_create < #{endTime} LIMIT 1000
+		DELETE FROM broker_metrics WHERE gmt_create < #{endTime} LIMIT #{limitSize}
 	]]>
 	</delete>
 </mapper>

kafka-manager-dao/src/main/resources/mapper/ClusterMetricsDao.xml

@@ -27,9 +27,9 @@
         </foreach>
     </insert>
 
-    <delete id="deleteBeforeTime" parameterType="java.util.Date">
+    <delete id="deleteBeforeTime" parameterType="java.util.Map">
         <![CDATA[
-		DELETE FROM cluster_metrics WHERE gmt_create < #{endTime} LIMIT 200
+		DELETE FROM cluster_metrics WHERE gmt_create < #{endTime} LIMIT #{limitSize}
 	]]>
     </delete>
 </mapper>

kafka-manager-dao/src/main/resources/mapper/TopicAppMetricsDao.xml

@@ -30,9 +30,9 @@
 	]]>
 	</select>
 
-	<delete id="deleteBeforeTime" parameterType="java.util.Date">
+	<delete id="deleteBeforeTime" parameterType="java.util.Map">
 		<![CDATA[
-		DELETE FROM topic_app_metrics WHERE gmt_create < #{endTime} LIMIT 3000
+		DELETE FROM topic_app_metrics WHERE gmt_create < #{endTime} LIMIT #{limitSize}
 	]]>
 	</delete>
 </mapper>

kafka-manager-dao/src/main/resources/mapper/TopicExpiredDao.xml

@@ -36,4 +36,8 @@
     <select id="getByTopic" parameterType="java.util.Map" resultMap="TopicExpiredMap">
         SELECT * FROM topic_expired WHERE cluster_id = #{clusterId} AND topic_name = #{topicName}
     </select>
+
+    <delete id="deleteByName" parameterType="java.util.Map">
+        DELETE FROM topic_expired WHERE cluster_id=#{clusterId} AND topic_name=#{topicName}
+    </delete>
 </mapper>

kafka-manager-dao/src/main/resources/mapper/TopicMetricsDao.xml

@@ -25,6 +25,7 @@
 		WHERE cluster_id = #{clusterId}
 			AND topic_name = #{topicName}
 			AND gmt_create BETWEEN #{startTime} AND #{endTime}
+		ORDER BY gmt_create
 	]]>
 	</select>
 
@@ -32,12 +33,13 @@
 		<![CDATA[
 		SELECT * FROM topic_metrics
 		WHERE cluster_id = #{clusterId} AND #{afterTime} <= gmt_create
+		ORDER BY gmt_create
 	]]>
 	</select>
 
-	<delete id="deleteBeforeTime" parameterType="java.util.Date">
+	<delete id="deleteBeforeTime" parameterType="java.util.Map">
 		<![CDATA[
-		DELETE FROM topic_metrics WHERE gmt_create < #{endTime} LIMIT 3000
+		DELETE FROM topic_metrics WHERE gmt_create < #{endTime} LIMIT #{limitSize}
 	]]>
 	</delete>
 </mapper>

kafka-manager-dao/src/main/resources/mapper/TopicRequestMetricsDao.xml

@@ -34,15 +34,9 @@
         ORDER BY gmt_create ASC
     </select>
 
-	<delete id="deleteBeforeTime" parameterType="java.util.Date">
+	<delete id="deleteBeforeTime" parameterType="java.util.Map">
 		<![CDATA[
-		DELETE FROM topic_request_time_metrics WHERE gmt_create < #{endTime} LIMIT 2000
+		DELETE FROM topic_request_time_metrics WHERE gmt_create < #{endTime} LIMIT #{limitSize}
-	]]>
-	</delete>
-
-	<delete id="deleteBeforeId" parameterType="java.lang.Long">
-		<![CDATA[
-		DELETE FROM topic_request_time_metrics WHERE id < #{id} LIMIT 20000
 	]]>
 	</delete>
 

kafka-manager-dao/src/main/resources/mapper/TopicThrottledMetricsDao.xml

@@ -54,9 +54,9 @@
         AND gmt_create > #{afterTime}
     </select>
 
-    <delete id="deleteBeforeTime" parameterType="java.util.Date">
+    <delete id="deleteBeforeTime" parameterType="java.util.Map">
         <![CDATA[
-		DELETE FROM topic_throttled_metrics WHERE gmt_create < #{endTime} LIMIT 3000
+		DELETE FROM topic_throttled_metrics WHERE gmt_create < #{endTime} LIMIT #{limitSize}
 	]]>
     </delete>
 </mapper>

kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java

@@ -31,16 +31,16 @@ public class BaseSessionSignOn extends AbstractSingleSignOn {
     private LdapAuthentication ldapAuthentication;
 
     //是否开启ldap验证
-    @Value(value = "${account.ldap.enabled:}")
+    @Value(value = "${account.ldap.enabled:false}")
     private Boolean accountLdapEnabled;
 
     //ldap自动注册的默认角色。请注意：它通常来说都是低权限角色
-    @Value(value = "${account.ldap.auth-user-registration-role:}")
+    @Value(value = "${account.ldap.auth-user-registration-role:normal}")
     private String authUserRegistrationRole;
 
     //ldap自动注册是否开启
-    @Value(value = "${account.ldap.auth-user-registration:}")
+    @Value(value = "${account.ldap.auth-user-registration:false}")
-    private boolean authUserRegistration;
+    private Boolean authUserRegistration;
 
     @Override
     public Result<String> loginAndGetLdap(HttpServletRequest request, HttpServletResponse response, LoginDTO dto) {

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCommunityTopicMetrics.java

@@ -30,16 +30,23 @@ public class CollectAndPublishCommunityTopicMetrics extends AbstractScheduledTas
 
     @Override
     protected List<ClusterDO> listAllTasks() {
+        // 获取需要进行指标采集的集群列表，这些集群将会被拆分到多台KM中进行执行。
         return clusterService.list();
     }
 
     @Override
     public void processTask(ClusterDO clusterDO) {
+        // 这里需要实现对clusterDO这个集群进行Topic指标采集的代码逻辑
+
+        // 进行Topic指标获取
         List<TopicMetrics> metricsList = getTopicMetrics(clusterDO.getId());
+
+        // 获取到Topic流量指标之后，发布一个事件，
         SpringTool.publish(new TopicMetricsCollectedEvent(this, clusterDO.getId(), metricsList));
     }
 
     private List<TopicMetrics> getTopicMetrics(Long clusterId) {
+        // 具体获取Topic流量指标的入口代码
         List<TopicMetrics> metricsList =
                 jmxService.getTopicMetrics(clusterId, KafkaMetricsCollections.TOPIC_METRICS_TO_DB, true);
         if (ValidateUtils.isEmptyList(metricsList)) {

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/delete/DeleteMetrics.java

@@ -1,15 +1,15 @@
 package com.xiaojukeji.kafka.manager.task.dispatch.metrics.delete;
 
-import com.xiaojukeji.kafka.manager.common.constant.Constant;
 import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
+import com.xiaojukeji.kafka.manager.common.utils.BackoffUtils;
 import com.xiaojukeji.kafka.manager.dao.*;
-import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils;
 import com.xiaojukeji.kafka.manager.task.component.AbstractScheduledTask;
 import com.xiaojukeji.kafka.manager.task.component.CustomScheduled;
 import com.xiaojukeji.kafka.manager.task.component.EmptyEntry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 
 import java.util.Arrays;
 import java.util.Date;
@@ -22,10 +22,7 @@ import java.util.List;
  */
 @CustomScheduled(name = "deleteMetrics", cron = "0 0/2 * * * ?", threadNum = 1)
 public class DeleteMetrics extends AbstractScheduledTask<EmptyEntry> {
-    private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
-
-    @Autowired
-    private ConfigUtils                 configUtils;
 
     @Autowired
     private TopicMetricsDao             topicMetricsDao;
@@ -45,6 +42,27 @@ public class DeleteMetrics extends AbstractScheduledTask<EmptyEntry> {
     @Autowired
     private TopicThrottledMetricsDao    topicThrottledMetricsDao;
 
+    @Value(value = "${task.metrics.delete-metrics.delete-limit-size:1000}")
+    private Integer deleteLimitSize;
+
+    @Value(value = "${task.metrics.delete-metrics.cluster-metrics-save-days:14}")
+    private Integer clusterMetricsSaveDays;
+
+    @Value(value = "${task.metrics.delete-metrics.broker-metrics-save-days:14}")
+    private Integer brokerMetricsSaveDays;
+
+    @Value(value = "${task.metrics.delete-metrics.topic-metrics-save-days:7}")
+    private Integer topicMetricsSaveDays;
+
+    @Value(value = "${task.metrics.delete-metrics.topic-request-time-metrics-save-days:7}")
+    private Integer topicRequestTimeMetricsSaveDays;
+
+    @Value(value = "${task.metrics.delete-metrics.topic-throttled-metrics-save-days:7}")
+    private Integer topicThrottledMetricsSaveDays;
+
+    @Value(value = "${task.metrics.delete-metrics.app-topic-metrics-save-days:7}")
+    private Integer appTopicMetricsSaveDays;
+
     @Override
     public List<EmptyEntry> listAllTasks() {
         EmptyEntry emptyEntry = new EmptyEntry();
@@ -54,78 +72,73 @@ public class DeleteMetrics extends AbstractScheduledTask<EmptyEntry> {
 
     @Override
     public void processTask(EmptyEntry entryEntry) {
-        if (Constant.INVALID_CODE.equals(configUtils.getMaxMetricsSaveDays())) {
-            // 无需数据删除
-            return;
-        }
-
         long startTime = System.currentTimeMillis();
         LOGGER.info("start delete metrics");
-        try {
+
-            deleteTopicMetrics();
+        // 数据量可能比较大，一次触发多删除几次
-        } catch (Exception e) {
+        for (int i = 0; i < 10; ++i) {
-            LOGGER.error("delete topic metrics failed.", e);
+            try {
+                boolean needReDelete = this.deleteCommunityTopicMetrics();
+                if (!needReDelete) {
+                    break;
+                }
+
+                // 暂停1000毫秒，避免删除太快导致DB出现问题
+                BackoffUtils.backoff(1000);
+            } catch (Exception e) {
+                LOGGER.error("delete community topic metrics failed.", e);
+            }
+        }
+
+        // 数据量可能比较大，一次触发多删除几次
+        for (int i = 0; i < 10; ++i) {
+            try {
+                boolean needReDelete = this.deleteDiDiTopicMetrics();
+                if (!needReDelete) {
+                    break;
+                }
+
+                // 暂停1000毫秒，避免删除太快导致DB出现问题
+                BackoffUtils.backoff(1000);
+            } catch (Exception e) {
+                LOGGER.error("delete didi topic metrics failed.", e);
+            }
         }
 
         try {
-            deleteTopicAppMetrics();
+            this.deleteClusterBrokerMetrics();
         } catch (Exception e) {
-            LOGGER.error("delete topic app metrics failed.", e);
+            LOGGER.error("delete cluster and broker metrics failed.", e);
         }
 
-        try {
-            deleteTopicRequestMetrics();
-        } catch (Exception e) {
-            LOGGER.error("delete topic request metrics failed.", e);
-        }
-
-        try {
-            deleteThrottledMetrics();
-        } catch (Exception e) {
-            LOGGER.error("delete topic throttled metrics failed.", e);
-        }
-
-        try {
-            deleteBrokerMetrics();
-        } catch (Exception e) {
-            LOGGER.error("delete broker metrics failed.", e);
-        }
-
-        try {
-            deleteClusterMetrics();
-        } catch (Exception e) {
-            LOGGER.error("delete cluster metrics failed.", e);
-        }
         LOGGER.info("finish delete metrics, costTime:{}ms.", System.currentTimeMillis() - startTime);
     }
 
-    private void deleteTopicMetrics() {
+    private boolean deleteCommunityTopicMetrics() {
-        Date endTime = new Date(System.currentTimeMillis() - configUtils.getMaxMetricsSaveDays() * 24 * 60 * 60 * 1000);
+        return topicMetricsDao.deleteBeforeTime(new Date(System.currentTimeMillis() - this.topicMetricsSaveDays * 24 * 60 * 60 * 1000), this.deleteLimitSize) >= this.deleteLimitSize;
-        topicMetricsDao.deleteBeforeTime(endTime);
     }
 
-    private void deleteTopicAppMetrics() {
+    private boolean deleteDiDiTopicMetrics() {
-        Date endTime = new Date(System.currentTimeMillis() - configUtils.getMaxMetricsSaveDays() * 24 * 60 * 60 * 1000);
+        boolean needReDelete = false;
-        topicAppMetricsDao.deleteBeforeTime(endTime);
+
+        if (topicAppMetricsDao.deleteBeforeTime(new Date(System.currentTimeMillis() - this.appTopicMetricsSaveDays * 24 * 60 * 60 * 1000), this.deleteLimitSize) >= this.deleteLimitSize) {
+            needReDelete = true;
+        }
+
+        if (topicRequestMetricsDao.deleteBeforeTime(new Date(System.currentTimeMillis() - this.topicRequestTimeMetricsSaveDays * 24 * 60 * 60 * 1000), this.deleteLimitSize) >= this.deleteLimitSize) {
+            needReDelete = true;
+        }
+
+        if (topicThrottledMetricsDao.deleteBeforeTime(new Date(System.currentTimeMillis() - this.topicThrottledMetricsSaveDays * 24 * 60 * 60 * 1000), this.deleteLimitSize) >= this.deleteLimitSize) {
+            needReDelete = true;
+        }
+
+        return needReDelete;
     }
 
-    private void deleteTopicRequestMetrics() {
+    private void deleteClusterBrokerMetrics() {
-        Date endTime = new Date(System.currentTimeMillis() - configUtils.getMaxMetricsSaveDays() * 24 * 60 * 60 * 1000);
+        brokerMetricsDao.deleteBeforeTime(new Date(System.currentTimeMillis() - this.brokerMetricsSaveDays * 24 * 60 * 60 * 1000), this.deleteLimitSize);
-        topicRequestMetricsDao.deleteBeforeTime(endTime);
-    }
 
-    private void deleteThrottledMetrics() {
+        clusterMetricsDao.deleteBeforeTime(new Date(System.currentTimeMillis() - this.clusterMetricsSaveDays * 24 * 60 * 60 * 1000), this.deleteLimitSize);
-        Date endTime = new Date(System.currentTimeMillis() - configUtils.getMaxMetricsSaveDays() * 24 * 60 * 60 * 1000);
-        topicThrottledMetricsDao.deleteBeforeTime(endTime);
-    }
-
-    private void deleteBrokerMetrics() {
-        Date endTime = new Date(System.currentTimeMillis() - configUtils.getMaxMetricsSaveDays() * 24 * 60 * 60 * 1000);
-        brokerMetricsDao.deleteBeforeTime(endTime);
-    }
-
-    private void deleteClusterMetrics() {
-        Date endTime = new Date(System.currentTimeMillis() - configUtils.getMaxMetricsSaveDays() * 24 * 60 * 60 * 1000);
-        clusterMetricsDao.deleteBeforeTime(endTime);
     }
 }

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/store/StoreBrokerMetrics.java

@@ -1,5 +1,6 @@
 package com.xiaojukeji.kafka.manager.task.dispatch.metrics.store;
 
+import com.xiaojukeji.kafka.manager.common.constant.Constant;
 import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
 import com.xiaojukeji.kafka.manager.common.constant.LogConstant;
 import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics;
@@ -54,8 +55,6 @@ public class StoreBrokerMetrics extends AbstractScheduledTask<ClusterDO> {
     @Autowired
     private AbstractHealthScoreStrategy healthScoreStrategy;
 
-    private static final Integer INSERT_BATCH_SIZE = 100;
-
     @Override
     protected List<ClusterDO> listAllTasks() {
         return clusterService.list();
@@ -72,7 +71,7 @@ public class StoreBrokerMetrics extends AbstractScheduledTask<ClusterDO> {
                     clusterDO.getId(),
                     MetricsConvertUtils.merge2BaseMetricsByAdd(brokerMetricsList))
             );
-        } catch (Throwable t) {
+        } catch (Exception t) {
             LOGGER.error("collect failed, clusterId:{}.", clusterDO.getId(), t);
         }
         long endTime = System.currentTimeMillis();
@@ -82,6 +81,11 @@ public class StoreBrokerMetrics extends AbstractScheduledTask<ClusterDO> {
                 startTime,
                 clusterMetricsList
         );
+
+        if (ValidateUtils.isEmptyList(doList)) {
+            return;
+        }
+
         clusterMetricsDao.batchAdd(doList);
     }
 
@@ -110,9 +114,15 @@ public class StoreBrokerMetrics extends AbstractScheduledTask<ClusterDO> {
                 MetricsConvertUtils.convertAndUpdateCreateTime2BrokerMetricsDOList(startTime, metricsList);
         int i = 0;
         do {
-            brokerMetricsDao.batchAdd(doList.subList(i, Math.min(i + INSERT_BATCH_SIZE, doList.size())));
+            List<BrokerMetricsDO> subDOList = doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size()));
-            i += INSERT_BATCH_SIZE;
+            if (ValidateUtils.isEmptyList(subDOList)) {
+                break;
+            }
+
+            brokerMetricsDao.batchAdd(subDOList);
+            i += Constant.BATCH_INSERT_SIZE;
         } while (i < doList.size());
+
         return metricsList;
     }
 

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/store/StoreDiDiAppTopicMetrics.java

@@ -17,7 +17,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.stereotype.Component;
 
 import java.util.*;
 
@@ -28,7 +27,7 @@ import java.util.*;
 @CustomScheduled(name = "storeDiDiAppTopicMetrics", cron = "41 0/1 * * * ?", threadNum = 5)
 @ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "app-topic-metrics-enabled", havingValue = "true", matchIfMissing = true)
 public class StoreDiDiAppTopicMetrics extends AbstractScheduledTask<ClusterDO> {
-    private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
 
     @Autowired
     private JmxService jmxService;
@@ -50,7 +49,7 @@ public class StoreDiDiAppTopicMetrics extends AbstractScheduledTask<ClusterDO> {
 
         try {
             getAndBatchAddTopicAppMetrics(startTime, clusterDO.getId());
-        } catch (Throwable t) {
+        } catch (Exception t) {
             LOGGER.error("save topic metrics failed, clusterId:{}.", clusterDO.getId(), t);
         }
     }
@@ -65,7 +64,12 @@ public class StoreDiDiAppTopicMetrics extends AbstractScheduledTask<ClusterDO> {
                 MetricsConvertUtils.convertAndUpdateCreateTime2TopicMetricsDOList(startTime, metricsList);
         int i = 0;
         do {
-            topicAppMetricsDao.batchAdd(doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size())));
+            List<TopicMetricsDO> subDOList = doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size()));
+            if (ValidateUtils.isEmptyList(subDOList)) {
+                return;
+            }
+
+            topicAppMetricsDao.batchAdd(subDOList);
             i += Constant.BATCH_INSERT_SIZE;
         } while (i < doList.size());
     }

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/store/StoreDiDiTopicRequestTimeMetrics.java

@@ -27,7 +27,7 @@ import java.util.*;
 @CustomScheduled(name = "storeDiDiTopicRequestTimeMetrics", cron = "51 0/1 * * * ?", threadNum = 5)
 @ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "topic-request-time-metrics-enabled", havingValue = "true", matchIfMissing = true)
 public class StoreDiDiTopicRequestTimeMetrics extends AbstractScheduledTask<ClusterDO> {
-    private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
 
     @Autowired
     private JmxService jmxService;
@@ -51,7 +51,7 @@ public class StoreDiDiTopicRequestTimeMetrics extends AbstractScheduledTask<Clus
             LOGGER.info("save topic metrics, clusterId:{}, start.", clusterDO.getId());
             getAndBatchAddTopicRequestTimeMetrics(startTime, clusterDO.getId());
             LOGGER.info("save topic metrics, clusterId:{}, end costTime:{}.", clusterDO.getId(), System.currentTimeMillis() - startTime);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             LOGGER.error("save topic metrics failed, clusterId:{}.", clusterDO.getId(), t);
         }
     }
@@ -69,7 +69,12 @@ public class StoreDiDiTopicRequestTimeMetrics extends AbstractScheduledTask<Clus
 
         int i = 0;
         do {
-            topicRequestMetricsDao.batchAdd(doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size())));
+            List<TopicMetricsDO> subDOList = doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size()));
+            if (ValidateUtils.isEmptyList(subDOList)) {
+                return;
+            }
+
+            topicRequestMetricsDao.batchAdd(subDOList);
             i += Constant.BATCH_INSERT_SIZE;
         } while (i < doList.size());
     }

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/listener/StoreCommunityTopicMetrics2DB.java

@@ -25,7 +25,7 @@ import java.util.List;
 @Component("storeCommunityTopicMetrics2DB")
 @ConditionalOnProperty(prefix = "custom.store-metrics-task.community", name = "topic-metrics-enabled", havingValue = "true", matchIfMissing = true)
 public class StoreCommunityTopicMetrics2DB implements ApplicationListener<TopicMetricsCollectedEvent> {
-    private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
 
     @Autowired
     private TopicMetricsDao topicMetricsDao;
@@ -40,17 +40,21 @@ public class StoreCommunityTopicMetrics2DB implements ApplicationListener<TopicM
 
         try {
             store2DB(System.currentTimeMillis(), metricsList);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             LOGGER.error("save topic metrics failed, clusterId:{}.", event.getClusterId(), t);
         }
     }
 
-    private void store2DB(Long startTime, List<TopicMetrics> metricsList) throws Exception {
+    private void store2DB(Long startTime, List<TopicMetrics> metricsList) {
-        List<TopicMetricsDO> doList =
+        List<TopicMetricsDO> doList = MetricsConvertUtils.convertAndUpdateCreateTime2TopicMetricsDOList(startTime, metricsList);
-                MetricsConvertUtils.convertAndUpdateCreateTime2TopicMetricsDOList(startTime, metricsList);
         int i = 0;
         do {
-            topicMetricsDao.batchAdd(doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size())));
+            List<TopicMetricsDO> subDOList = doList.subList(i, Math.min(i + Constant.BATCH_INSERT_SIZE, doList.size()));
+            if (ValidateUtils.isEmptyList(subDOList)) {
+                return;
+            }
+
+            topicMetricsDao.batchAdd(subDOList);
             i += Constant.BATCH_INSERT_SIZE;
         } while (i < doList.size());
     }

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/listener/StoreTopicThrottledMetrics2DB.java

@@ -22,7 +22,7 @@ import java.util.*;
  * @date 20/9/24
  */
 @Component("storeTopicThrottledMetrics2DB")
-@ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "topic-throttled-metrics", havingValue = "true", matchIfMissing = true)
+@ConditionalOnProperty(prefix = "custom.store-metrics-task.didi", name = "topic-throttled-metrics-enabled", havingValue = "true", matchIfMissing = true)
 public class StoreTopicThrottledMetrics2DB implements ApplicationListener<TopicThrottledMetricsCollectedEvent> {
     private final static Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER);
 

kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/listener/biz/RegionCreatedListener.java

@@ -0,0 +1,38 @@
+package com.xiaojukeji.kafka.manager.task.listener.biz;
+
+import com.xiaojukeji.kafka.manager.common.events.RegionCreatedEvent;
+import com.xiaojukeji.kafka.manager.task.dispatch.biz.CalRegionCapacity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationListener;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Component;
+
+/**
+ * Region创建监听器,
+ * TODO 后续需要将其移动到core模块
+ * @author zengqiao
+ * @date 22/01/11
+ */
+@Component
+public class RegionCreatedListener implements ApplicationListener<RegionCreatedEvent> {
+    private static final Logger logger = LoggerFactory.getLogger(RegionCreatedListener.class);
+
+    @Autowired
+    private CalRegionCapacity calRegionCapacity;
+
+    @Async
+    @Override
+    public void onApplicationEvent(RegionCreatedEvent event) {
+        try {
+            logger.info("cal region capacity started when region created, regionDO:{}.", event.getRegionDO());
+
+            calRegionCapacity.processTask(event.getRegionDO());
+
+            logger.info("cal region capacity finished when region created, regionDO:{}.", event.getRegionDO());
+        } catch (Exception e) {
+            logger.error("cal region capacity failed when region created, regionDO:{}.", event.getRegionDO(), e);
+        }
+    }
+}

kafka-manager-web/pom.xml

@@ -19,7 +19,8 @@
         <springframework.boot.version>2.1.1.RELEASE</springframework.boot.version>
         <spring-version>5.1.3.RELEASE</spring-version>
         <failOnMissingWebXml>false</failOnMissingWebXml>
-        <tomcat.version>8.5.66</tomcat.version>
+        <tomcat.version>8.5.72</tomcat.version>
+        <log4j2.version>2.16.0</log4j2.version>
     </properties>
 
     <dependencies>
@@ -109,8 +110,10 @@
     </dependencies>
 
     <build>
+        <finalName>kafka-manager</finalName>
         <plugins>
             <plugin>
+
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
                 <version>${springframework.boot.version}</version>
@@ -121,6 +124,7 @@
                         </goals>
                     </execution>
                 </executions>
+
             </plugin>
         </plugins>
     </build>

kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/config/SwaggerConfig.java

@@ -39,10 +39,9 @@ public class SwaggerConfig implements WebMvcConfigurer {
 
     private ApiInfo apiInfo() {
         return new ApiInfoBuilder()
-                .title("Logi-KafkaManager 接口文档")
+                .title("LogiKM接口文档")
-                .description("欢迎使用滴滴Logi-KafkaManager")
+                .description("欢迎使用滴滴LogiKM")
-                .contact("huangyiminghappy@163.com")
+                .version("2.5.0")
-                .version("2.2.0")
                 .build();
     }
 

kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/OrderConverter.java

@@ -1,15 +1,16 @@
 package com.xiaojukeji.kafka.manager.web.converters;
 
-import com.xiaojukeji.kafka.manager.common.entity.ao.account.Account;
 import com.xiaojukeji.kafka.manager.bpm.common.OrderResult;
+import com.xiaojukeji.kafka.manager.bpm.common.OrderStatusEnum;
 import com.xiaojukeji.kafka.manager.bpm.common.entry.BaseOrderDetailData;
+import com.xiaojukeji.kafka.manager.common.entity.ao.account.Account;
+import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.common.AccountVO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.normal.order.OrderResultVO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.normal.order.OrderVO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.normal.order.detail.OrderDetailBaseVO;
 import com.xiaojukeji.kafka.manager.common.utils.CopyUtils;
 import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
-import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -41,7 +42,9 @@ public class OrderConverter {
         }
         OrderVO orderVO = new OrderVO();
         CopyUtils.copyProperties(orderVO, orderDO);
-        orderVO.setGmtTime(orderDO.getGmtCreate());
+        if (OrderStatusEnum.WAIT_DEAL.getCode().equals(orderDO.getStatus())) {
+            orderVO.setGmtHandle(null);
+        }
         return orderVO;
     }
 

kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ReassignModelConverter.java

@@ -95,12 +95,21 @@ public class ReassignModelConverter {
         vo.setBeginTime(0L);
         vo.setEndTime(0L);
 
+        StringBuilder clusterAndTopicName = new StringBuilder();
+
         Integer completedTopicNum = 0;
         Set<Integer> statusSet = new HashSet<>();
         for (ReassignTaskDO elem: doList) {
             vo.setGmtCreate(elem.getGmtCreate().getTime());
             vo.setOperator(elem.getOperator());
             vo.setDescription(elem.getDescription());
+
+            if (clusterAndTopicName.length() == 0) {
+                clusterAndTopicName.append("-").append(elem.getClusterId()).append("-").append(elem.getTopicName());
+            } else {
+                clusterAndTopicName.append("等");
+            }
+
             if (TaskStatusReassignEnum.isFinished(elem.getStatus())) {
                 completedTopicNum += 1;
                 statusSet.add(elem.getStatus());
@@ -114,6 +123,9 @@ public class ReassignModelConverter {
             vo.setBeginTime(elem.getBeginTime().getTime());
         }
 
+        // 任务名称上，增加展示集群ID和Topic名称，多个时，仅展示第一个. PR from Hongten
+        vo.setTaskName(String.format("%s 数据迁移任务%s", DateUtils.getFormattedDate(taskId), clusterAndTopicName.toString()));
+
         // 任务整体状态
         if (statusSet.contains(TaskStatusReassignEnum.RUNNING.getCode())) {
             vo.setStatus(TaskStatusReassignEnum.RUNNING.getCode());

kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicMineConverter.java

@@ -29,6 +29,7 @@ public class TopicMineConverter {
             vo.setClusterName(data.getLogicalClusterName());
             vo.setBytesIn(data.getBytesIn());
             vo.setBytesOut(data.getBytesOut());
+            vo.setDescription(data.getDescription());
             voList.add(vo);
         }
         return voList;

kafka-manager-web/src/main/resources/application.yml

@@ -9,6 +9,8 @@ server:
 spring:
   application:
     name: kafkamanager
+  profiles:
+    active: dev
   datasource:
     kafka-manager:
       jdbc-url: jdbc:mysql://localhost:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8
@@ -18,8 +20,6 @@ spring:
   main:
     allow-bean-definition-overriding: true
 
-  profiles:
-    active: dev
   servlet:
     multipart:
       max-file-size: 100MB
@@ -30,25 +30,31 @@ logging:
 
 custom:
   idc: cn
-  jmx:
-    max-conn: 10 # 2.3版本配置不在这个地方生效
   store-metrics-task:
     community:
       broker-metrics-enabled: true
       topic-metrics-enabled: true
-    didi:
+    didi: # 滴滴Kafka特有的指标
       app-topic-metrics-enabled: false
       topic-request-time-metrics-enabled: false
-      topic-throttled-metrics: false
+      topic-throttled-metrics-enabled: false
-    save-days: 7
 
-# 任务相关的开关
+# 任务相关的配置
 task:
   op:
     sync-topic-enabled: false # 未落盘的Topic定期同步到DB中
     order-auto-exec:          # 工单自动化审批线程的开关
       topic-enabled: false      # Topic工单自动化审批开关, false:关闭自动化审批, true:开启
       app-enabled: false        # App工单自动化审批开关, false:关闭自动化审批, true:开启
+  metrics:
+    delete-metrics:
+      delete-limit-size: 1000
+      cluster-metrics-save-days: 14                   # 集群指标保存天数
+      broker-metrics-save-days: 14                    # Broker指标保存天数
+      topic-metrics-save-days: 7                      # Topic指标保存天数
+      topic-request-time-metrics-save-days: 7         # Topic请求耗时指标保存天数
+      topic-throttled-metrics-save-days: 7            # Topic限流指标保存天数
+      app-topic-metrics-save-days: 7                  # App+Topic指标保存天数
 
 account:
   ldap:

pom.xml

@@ -16,9 +16,9 @@
     </parent>
 
     <properties>
-        <kafka-manager.revision>2.4.2-SNAPSHOT</kafka-manager.revision>
+        <kafka-manager.revision>2.5</kafka-manager.revision>
-        <swagger2.version>2.7.0</swagger2.version>
+        <swagger2.version>2.9.2</swagger2.version>
-        <swagger.version>1.5.13</swagger.version>
+        <swagger.version>1.5.21</swagger.version>
 
         <maven.test.skip>true</maven.test.skip>
         <downloadSources>true</downloadSources>
@@ -26,7 +26,9 @@
         <java_target_version>1.8</java_target_version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <file_encoding>UTF-8</file_encoding>
-        <tomcat.version>8.5.66</tomcat.version>
+        <tomcat.version>8.5.72</tomcat.version>
+        <log4j2.version>2.16.0</log4j2.version>
+        <maven-assembly-plugin.version>3.0.0</maven-assembly-plugin.version>
     </properties>
 
     <modules>
@@ -42,6 +44,7 @@
         <module>kafka-manager-extends/kafka-manager-openapi</module>
         <module>kafka-manager-task</module>
         <module>kafka-manager-web</module>
+        <module>distribution</module>
     </modules>
 
     <dependencyManagement>
@@ -229,6 +232,25 @@
                 <artifactId>minio</artifactId>
                 <version>7.1.0</version>
             </dependency>
+
+            <dependency>
+                <groupId>org.projectlombok</groupId>
+                <artifactId>lombok</artifactId>
+                <version>1.18.2</version>
+                <scope>provided</scope>
+            </dependency>
         </dependencies>
     </dependencyManagement>
+
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>${maven-assembly-plugin.version}</version>
+            </plugin>
+        </plugins>
+    </build>
 </project>