diff --git a/README.md b/README.md index 885c2a10..2e284136 100644 --- a/README.md +++ b/README.md @@ -75,19 +75,28 @@ ![image](https://user-images.githubusercontent.com/5287750/111266722-e531d800-8665-11eb-9242-3484da5a3099.png) 微信加群：添加mike_zhangliang的微信号备注Logi加群或关注公众号 云原生可观测性 回复 "Logi加群" +## 4 知识星球 -## 4 项目成员 +![image](https://user-images.githubusercontent.com/51046167/140718512-5ab1b336-5c48-46c0-90bd-44b5c7e168c8.png) -### 4.1 内部核心人员 +✅知识星球首个【Kafka中文社区】，内测期免费加入～https://z.didi.cn/5gSF9 +有问必答~！ +互动有礼~！ +1600+群友一起共建国内最专业的【Kafka中文社区】 +PS:提问请尽量把问题一次性描述清楚，并告知环境信息情况哦～！如使用版本、操作步骤、报错/警告信息等，方便嘉宾们快速解答～ + +## 5 项目成员 + +### 5.1 内部核心人员 `iceyuhui`、`liuyaguang`、`limengmonty`、`zhangliangmike`、`nullhuangyiming`、`zengqiao`、`eilenexuzhe`、`huangjiaweihjw`、`zhaoyinrui`、`marzkonglingxu`、`joysunchao` -### 4.2 外部贡献者 +### 5.2 外部贡献者 `fangjunyu`、`zhoutaiyang` -## 5 协议 +## 6 协议 -`kafka-manager`基于`Apache-2.0`协议进行分发和使用，更多信息参见[协议文件](./LICENSE) +`LogiKM`基于`Apache-2.0`协议进行分发和使用，更多信息参见[协议文件](./LICENSE) diff --git a/build.sh b/build.sh deleted file mode 100644 index b07c6623..00000000 --- a/build.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -workspace=$(cd $(dirname $0) && pwd -P) -cd $workspace - -## constant -OUTPUT_DIR=./output -KM_VERSION=2.4.2 -APP_NAME=kafka-manager -APP_DIR=${APP_NAME}-${KM_VERSION} - -MYSQL_TABLE_SQL_FILE=./docs/install_guide/create_mysql_table.sql -CONFIG_FILE=./kafka-manager-web/src/main/resources/application.yml - -## function -function build() { - # 编译命令 - mvn -U clean package -Dmaven.test.skip=true - - local sc=$? - if [ $sc -ne 0 ];then - ## 编译失败, 退出码为 非0 - echo "$APP_NAME build error" - exit $sc - else - echo "$APP_NAME build ok" - fi -} - -function make_output() { - # 新建output目录 - rm -rf ${OUTPUT_DIR} &>/dev/null - mkdir -p ${OUTPUT_DIR}/${APP_DIR} &>/dev/null - - # 填充output目录, output内的内容 - ( - cp -rf ${MYSQL_TABLE_SQL_FILE} ${OUTPUT_DIR}/${APP_DIR} && # 拷贝 sql 初始化脚本 至output目录 - cp -rf ${CONFIG_FILE} ${OUTPUT_DIR}/${APP_DIR} && # 拷贝 application.yml 至output目录 - - # 拷贝程序包到output路径 - cp kafka-manager-web/target/kafka-manager-web-${KM_VERSION}-SNAPSHOT.jar ${OUTPUT_DIR}/${APP_DIR}/${APP_NAME}.jar - echo -e "make output ok." - ) || { echo -e "make output error"; exit 2; } # 填充output目录失败后, 退出码为 非0 -} - -function make_package() { - # 压缩output目录 - ( - cd ${OUTPUT_DIR} && tar cvzf ${APP_DIR}.tar.gz ${APP_DIR} - echo -e "make package ok." - ) || { echo -e "make package error"; exit 2; } # 压缩output目录失败后, 退出码为 非0 -} - -########################################## -## main -## 其中, -## 1.进行编译 -## 2.生成部署包output -## 3.生成tar.gz压缩包 -########################################## - -# 1.进行编译 -build - -# 2.生成部署包output -make_output - -# 3.生成tar.gz压缩包 -make_package - -# 编译成功 -echo -e "build done" -exit 0 \ No newline at end of file diff --git a/distribution/bin/shutdown.sh b/distribution/bin/shutdown.sh new file mode 100644 index 00000000..fdf2d01c --- /dev/null +++ b/distribution/bin/shutdown.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cd `dirname $0`/../target +target_dir=`pwd` + +pid=`ps ax | grep -i 'kafka-manager' | grep ${target_dir} | grep java | grep -v grep | awk '{print $1}'` +if [ -z "$pid" ] ; then + echo "No kafka-manager running." + exit -1; +fi + +echo "The kafka-manager (${pid}) is running..." + +kill ${pid} + +echo "Send shutdown request to kafka-manager (${pid}) OK" diff --git a/distribution/bin/startup.sh b/distribution/bin/startup.sh new file mode 100644 index 00000000..ead6cde9 --- /dev/null +++ b/distribution/bin/startup.sh @@ -0,0 +1,81 @@ +error_exit () +{ + echo "ERROR: $1 !!" + exit 1 +} + +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/usr/java +[ ! -e "$JAVA_HOME/bin/java" ] && unset JAVA_HOME + +if [ -z "$JAVA_HOME" ]; then + if $darwin; then + + if [ -x '/usr/libexec/java_home' ] ; then + export JAVA_HOME=`/usr/libexec/java_home` + + elif [ -d "/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" ]; then + export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" + fi + else + JAVA_PATH=`dirname $(readlink -f $(which javac))` + if [ "x$JAVA_PATH" != "x" ]; then + export JAVA_HOME=`dirname $JAVA_PATH 2>/dev/null` + fi + fi + if [ -z "$JAVA_HOME" ]; then + error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)! jdk8 or later is better!" + fi +fi + + + + +export WEB_SERVER="kafka-manager" +export JAVA_HOME +export JAVA="$JAVA_HOME/bin/java" +export BASE_DIR=`cd $(dirname $0)/..; pwd` +export CUSTOM_SEARCH_LOCATIONS=file:${BASE_DIR}/conf/ + + +#=========================================================================================== +# JVM Configuration +#=========================================================================================== + +JAVA_OPT="${JAVA_OPT} -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof" + +## jdk版本高的情况 有些 参数废弃了 +JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p') +if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then + JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/km_gc.log:time,tags:filecount=10,filesize=102400" +else + JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext" + JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/km_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M" + +fi + +JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/${WEB_SERVER}.jar" +JAVA_OPT="${JAVA_OPT} --spring.config.additional-location=${CUSTOM_SEARCH_LOCATIONS}" +JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/logback-spring.xml" +JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288" + + + +if [ ! -d "${BASE_DIR}/logs" ]; then + mkdir ${BASE_DIR}/logs +fi + +echo "$JAVA ${JAVA_OPT}" + +# check the start.out log output file +if [ ! -f "${BASE_DIR}/logs/start.out" ]; then + touch "${BASE_DIR}/logs/start.out" +fi +# start +echo -e "---- 启动脚本 ------\n $JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 & + + +nohup $JAVA ${JAVA_OPT} >> ${BASE_DIR}/logs/start.out 2>&1 & + +echo "${WEB_SERVER} is starting，you can check the ${BASE_DIR}/logs/start.out" diff --git a/distribution/conf/application.yml b/distribution/conf/application.yml new file mode 100644 index 00000000..6b78c104 --- /dev/null +++ b/distribution/conf/application.yml @@ -0,0 +1,28 @@ + +## kafka-manager的配置文件，该文件中的配置会覆盖默认配置 +## 下面的配置信息基本就是jar中的 application.yml默认配置了; +## 可以只修改自己变更的配置,其他的删除就行了; 比如只配置一下mysql + + +server: + port: 8080 + tomcat: + accept-count: 1000 + max-connections: 10000 + max-threads: 800 + min-spare-threads: 100 + +spring: + application: + name: kafkamanager + profiles: + active: dev + datasource: + kafka-manager: + jdbc-url: jdbc:mysql://localhost:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8 + username: root + password: 123456 + driver-class-name: com.mysql.cj.jdbc.Driver + main: + allow-bean-definition-overriding: true + diff --git a/distribution/conf/application.yml.example b/distribution/conf/application.yml.example new file mode 100644 index 00000000..31c135af --- /dev/null +++ b/distribution/conf/application.yml.example @@ -0,0 +1,116 @@ + +## kafka-manager的配置文件，该文件中的配置会覆盖默认配置 +## 下面的配置信息基本就是jar中的 application.yml默认配置了; +## 可以只修改自己变更的配置,其他的删除就行了; 比如只配置一下mysql + + +server: + port: 8080 + tomcat: + accept-count: 1000 + max-connections: 10000 + max-threads: 800 + min-spare-threads: 100 + +spring: + application: + name: kafkamanager + profiles: + active: dev + datasource: + kafka-manager: + jdbc-url: jdbc:mysql://localhost:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8 + username: root + password: 123456 + driver-class-name: com.mysql.cj.jdbc.Driver + main: + allow-bean-definition-overriding: true + + + servlet: + multipart: + max-file-size: 100MB + max-request-size: 100MB + +logging: + config: classpath:logback-spring.xml + +custom: + idc: cn # 部署的数据中心, 忽略该配置, 后续会进行删除 + jmx: + max-conn: 10 # 2.3版本配置不在这个地方生效 + store-metrics-task: + community: + broker-metrics-enabled: true # 社区部分broker metrics信息收集开关, 关闭之后metrics信息将不会进行收集及写DB + topic-metrics-enabled: true # 社区部分topic的metrics信息收集开关, 关闭之后metrics信息将不会进行收集及写DB + didi: + app-topic-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭 + topic-request-time-metrics-enabled: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭 + topic-throttled-metrics: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭 + save-days: 7 #指标在DB中保持的天数，-1表示永久保存，7表示保存近7天的数据 + +# 任务相关的开关 +task: + op: + sync-topic-enabled: false # 未落盘的Topic定期同步到DB中 + order-auto-exec: # 工单自动化审批线程的开关 + topic-enabled: false # Topic工单自动化审批开关, false:关闭自动化审批, true:开启 + app-enabled: false # App工单自动化审批开关, false:关闭自动化审批, true:开启 + +# ldap相关的配置 +account: + ldap: + enabled: false + url: ldap://127.0.0.1:389/ + basedn: dc=tsign,dc=cn + factory: com.sun.jndi.ldap.LdapCtxFactory + filter: sAMAccountName + security: + authentication: simple + principal: cn=admin,dc=tsign,dc=cn + credentials: admin + auth-user-registration: true + auth-user-registration-role: normal + +# 集群升级部署相关的功能，需要配合夜莺及S3进行使用 +kcm: + enabled: false + s3: + endpoint: s3.didiyunapi.com + access-key: 1234567890 + secret-key: 0987654321 + bucket: logi-kafka + n9e: + base-url: http://127.0.0.1:8004 + user-token: 12345678 + timeout: 300 + account: root + script-file: kcm_script.sh + +# 监控告警相关的功能，需要配合夜莺进行使用 +# enabled: 表示是否开启监控告警的功能, true: 开启, false: 不开启 +# n9e.nid: 夜莺的节点ID +# n9e.user-token: 用户的密钥，在夜莺的个人设置中 +# n9e.mon.base-url: 监控地址 +# n9e.sink.base-url: 数据上报地址 +# n9e.rdb.base-url: 用户资源中心地址 + +monitor: + enabled: false + n9e: + nid: 2 + user-token: 1234567890 + mon: + base-url: http://127.0.0.1:8000 # 夜莺v4版本，默认端口统一调整为了8000 + sink: + base-url: http://127.0.0.1:8000 # 夜莺v4版本，默认端口统一调整为了8000 + rdb: + base-url: http://127.0.0.1:8000 # 夜莺v4版本，默认端口统一调整为了8000 + + +notify: # 通知的功能 + kafka: # 默认通知发送到kafka的指定Topic中 + cluster-id: 95 # Topic的集群ID + topic-name: didi-kafka-notify # Topic名称 + order: # 部署的KM的地址 + detail-url: http://127.0.0.1 \ No newline at end of file diff --git a/docs/install_guide/create_mysql_table.sql b/distribution/conf/create_mysql_table.sql similarity index 100% rename from docs/install_guide/create_mysql_table.sql rename to distribution/conf/create_mysql_table.sql diff --git a/distribution/conf/logback-spring.xml b/distribution/conf/logback-spring.xml new file mode 100644 index 00000000..c1c16136 --- /dev/null +++ b/distribution/conf/logback-spring.xml @@ -0,0 +1,215 @@ + + + logback + + + + + + + + + + + + + + info + + + ${CONSOLE_LOG_PATTERN} + UTF-8 + + + + + + + + + ${log.path}/log_debug.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + + + ${log.path}/log_debug_%d{yyyy-MM-dd}.%i.log + + 100MB + + + 7 + + + + debug + ACCEPT + DENY + + + + + + + ${log.path}/log_info.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + + + ${log.path}/log_info_%d{yyyy-MM-dd}.%i.log + + 100MB + + + 7 + + + + info + ACCEPT + DENY + + + + + + + ${log.path}/log_warn.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + + ${log.path}/log_warn_%d{yyyy-MM-dd}.%i.log + + 100MB + + + 7 + + + + warn + ACCEPT + DENY + + + + + + + + ${log.path}/log_error.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + + ${log.path}/log_error_%d{yyyy-MM-dd}.%i.log + + 100MB + + + 7 + + + + ERROR + ACCEPT + DENY + + + + + + ${log.path}/metrics/collector_metrics.log + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + ${log.path}/metrics/collector_metrics_%d{yyyy-MM-dd}.%i.log + + 100MB + + 3 + + + + + + ${log.path}/metrics/api_metrics.log + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + ${log.path}/metrics/api_metrics_%d{yyyy-MM-dd}.%i.log + + 100MB + + 3 + + + + + + ${log.path}/metrics/scheduled_tasks.log + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + UTF-8 + + + ${log.path}/metrics/scheduled_tasks_%d{yyyy-MM-dd}.%i.log + + 100MB + + 5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/distribution/pom.xml b/distribution/pom.xml new file mode 100644 index 00000000..6b61525c --- /dev/null +++ b/distribution/pom.xml @@ -0,0 +1,64 @@ + + + + + + kafka-manager + com.xiaojukeji.kafka + ${kafka-manager.revision} + + + 4.0.0 + + distribution + distribution + pom + + + + ${project.groupId} + kafka-manager-web + ${kafka-manager.revision} + + + + + + + release-kafka-manager + + + ${project.groupId} + kafka-manager-web + ${kafka-manager.revision} + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + release-km.xml + + posix + + + + make-assembly + install + + single + + + + + + kafka-manager + + + + diff --git a/distribution/readme.md b/distribution/readme.md new file mode 100644 index 00000000..9d40efa4 --- /dev/null +++ b/distribution/readme.md @@ -0,0 +1,22 @@ +## 说明 + +### 1.创建mysql数据库文件 +> conf/create_mysql_table.sql + +### 2. 修改配置文件 +> conf/application.yml.example +> 请将application.yml.example 复制一份改名为application.yml； +> 并放在同级目录下(conf/); 并修改成自己的配置 +> 这里的优先级比jar包内配置文件的默认值高; +> + +### 3.启动/关闭kafka-manager +> sh bin/startup.sh 启动 +> +> sh shutdown.sh 关闭 +> + + +### 4.升级jar包 +> 如果是升级, 可以看看文件 `upgrade_config.md` 的配置变更历史; +> \ No newline at end of file diff --git a/distribution/release-km.xml b/distribution/release-km.xml new file mode 100755 index 00000000..d1b0db1c --- /dev/null +++ b/distribution/release-km.xml @@ -0,0 +1,51 @@ + + + + ${project.version} + true + + dir + tar.gz + zip + + + + + conf/** + + + + + + bin/* + + 0755 + + + + + + + readme.md + readme.md + + + upgrade_config.md + upgrade_config.md + + + + ../kafka-manager-web/target/kafka-manager.jar + target/ + + + + + + true + + com.xiaojukeji.kafka:kafka-manager-web + + + + diff --git a/distribution/upgrade_config.md b/distribution/upgrade_config.md new file mode 100644 index 00000000..5f976042 --- /dev/null +++ b/distribution/upgrade_config.md @@ -0,0 +1,42 @@ + +## 版本升级配置变更 +> 本文件 从 V2.2.0 开始记录; 如果配置有变更则会填写到下文中; 如果没有,则表示无变更; +> 当您从一个很低的版本升级时候,应该依次执行中间有过变更的sql脚本 + + + +**一站式`Apache Kafka`集群指标监控与运维管控平台** + +--- + +### 1.升级至`V2.2.0`版本 + +#### 1.mysql变更 + +`2.2.0`版本在`cluster`表及`logical_cluster`各增加了一个字段，因此需要执行下面的sql进行字段的增加。 + +```sql +# 往cluster表中增加jmx_properties字段, 这个字段会用于存储jmx相关的认证以及配置信息 +ALTER TABLE `cluster` ADD COLUMN `jmx_properties` TEXT NULL COMMENT 'JMX配置' AFTER `security_properties`; + +# 往logical_cluster中增加identification字段, 同时数据和原先name数据相同, 最后增加一个唯一键. +# 此后, name字段还是表示集群名称, 而identification字段表示的是集群标识, 只能是字母数字及下划线组成, +# 数据上报到监控系统时, 集群这个标识采用的字段就是identification字段, 之前使用的是name字段. +ALTER TABLE `logical_cluster` ADD COLUMN `identification` VARCHAR(192) NOT NULL DEFAULT '' COMMENT '逻辑集群标识' AFTER `name`; + +UPDATE `logical_cluster` SET `identification`=`name` WHERE id>=0; + +ALTER TABLE `logical_cluster` ADD INDEX `uniq_identification` (`identification` ASC); +``` + +### 升级至`2.3.0`版本 + +#### 1.mysql变更 +`2.3.0`版本在`gateway_config`表增加了一个描述说明的字段，因此需要执行下面的sql进行字段的增加。 + +```sql +ALTER TABLE `gateway_config` +ADD COLUMN `description` TEXT NULL COMMENT '描述信息' AFTER `version`; +``` + + diff --git a/docs/dev_guide/assets/increase_the_indicators_reported_to_monitor_system/collect_topic_metrics.jpg b/docs/dev_guide/assets/increase_the_indicators_reported_to_monitor_system/collect_topic_metrics.jpg new file mode 100644 index 00000000..c2613d0a Binary files /dev/null and b/docs/dev_guide/assets/increase_the_indicators_reported_to_monitor_system/collect_topic_metrics.jpg differ diff --git a/docs/dev_guide/assets/increase_the_indicators_reported_to_monitor_system/sink_metrcis.png b/docs/dev_guide/assets/increase_the_indicators_reported_to_monitor_system/sink_metrcis.png new file mode 100644 index 00000000..ba27bc1c Binary files /dev/null and b/docs/dev_guide/assets/increase_the_indicators_reported_to_monitor_system/sink_metrcis.png differ diff --git a/docs/dev_guide/如何增加上报监控系统指标.md b/docs/dev_guide/如何增加上报监控系统指标.md new file mode 100644 index 00000000..f1ed9260 --- /dev/null +++ b/docs/dev_guide/如何增加上报监控系统指标.md @@ -0,0 +1,53 @@ + +--- + +![kafka-manager-logo](../assets/images/common/logo_name.png) + +**一站式`Apache Kafka`集群指标监控与运维管控平台** + +--- + +# 如何增加上报监控系统指标？ + +## 0、前言 + +LogiKM是 **一站式`Apache Kafka`集群指标监控与运维管控平台** ，当前会将消费Lag，Topic流量等指标上报到监控系统中，从而方便用户在监控系统中对这些指标配置监控告警规则，进而达到监控自身客户端是否正常的目的。 + +那么，如果我们想增加一个新的监控指标，应该如何做呢，比如我们想监控Broker的流量，监控Broker的存活信息，监控集群Controller个数等等。 + +在具体介绍之前，我们大家都知道，Kafka监控相关的信息，基本都存储于Broker、Jmx以及ZK中。当前LogiKM也已经具备从这三个地方获取数据的基本能力，因此基于LogiKM我们再获取其他指标，总体上还是非常方便的。 + +这里我们就以已经获取到的Topic流量信息为例，看LogiKM如何实现Topic指标的获取并上报的。 + +--- + +## 1、确定指标位置 + +基于对Kafka的了解，我们知道Topic流量信息这个指标是存储于Jmx中的，因此我们需要从Jmx中获取。大家如果对于自己所需要获取的指标存储在何处不太清楚的，可以加入我们维护的Kafka中文社区(README中有二维码)中今天沟通交流。 + +--- + +## 2、指标获取 + +Topic流量指标的获取详细见图中说明。 + +![Topic流量指标采集说明](./assets/increase_the_indicators_reported_to_monitor_system/collect_topic_metrics.jpg) + +--- + +## 3、指标上报 + +上一步我们已经采集到Topic流量指标了，下一步就是将该指标上报到监控系统，这块只需要按照监控系统要求的格式，将数据上报即可。 + +LogiKM中有一个monitor模块，具体的如下图所示： + +![指标上报](./assets/increase_the_indicators_reported_to_monitor_system/sink_metrcis.png) + + +## 4、补充说明 + +监控系统对接的相关内容见： + +[监控系统集成](./monitor_system_integrate_with_self.md) + +[监控系统集成例子——集成夜莺](./monitor_system_integrate_with_n9e.md) diff --git a/docs/install_guide/config_description.md b/docs/install_guide/config_description.md index 8f8fc0be..f615eacb 100644 --- a/docs/install_guide/config_description.md +++ b/docs/install_guide/config_description.md @@ -58,6 +58,9 @@ custom: task: op: sync-topic-enabled: false # 未落盘的Topic定期同步到DB中 + order-auto-exec: # 工单自动化审批线程的开关 + topic-enabled: false # Topic工单自动化审批开关, false:关闭自动化审批, true:开启 + app-enabled: false # App工单自动化审批开关, false:关闭自动化审批, true:开启 account: # ldap相关的配置, 社区版本暂时支持不够完善，可以先忽略，欢迎贡献代码对这块做优化 ldap: diff --git a/docs/install_guide/install_guide_cn.md b/docs/install_guide/install_guide_cn.md index 9a4a415b..0130bd55 100644 --- a/docs/install_guide/install_guide_cn.md +++ b/docs/install_guide/install_guide_cn.md @@ -31,17 +31,23 @@ **2、源代码进行打包** -下载好代码之后，进入`Logi-KafkaManager`的主目录，执行`sh build.sh`命令即可，执行完成之后会在`output/kafka-manager-xxx`目录下面生成一个jar包。 +下载好代码之后，进入`Logi-KafkaManager`的主目录，执行`mvn -Prelease-kafka-manager -Dmaven.test.skip=true clean install -U `命令即可， +执行完成之后会在`distribution/target`目录下面生成一个`kafka-manager-*.tar.gz`。 +和一个`kafka-manager-*.zip` 文件,随便任意一个压缩包都可以; +当然此时同级目录有一个已经解压好的文件夹; -对于`windows`环境的用户，估计执行不了`sh build.sh`命令，因此可以直接执行`mvn install`，然后在`kafka-manager-web/target`目录下生成一个kafka-manager-web-xxx.jar的包。 -获取到jar包之后，我们继续下面的步骤。 --- -## 3、MySQL-DB初始化 +## 3. 解压安装包 +解压完成后; 在文件目录中可以看到有`kafka-manager/conf/create_mysql_table.sql` 有个mysql初始化文件 +先初始化DB -执行[create_mysql_table.sql](create_mysql_table.sql)中的SQL命令，从而创建所需的MySQL库及表，默认创建的库名是`logi_kafka_manager`。 + +## 4、MySQL-DB初始化 + +执行[create_mysql_table.sql](../../distribution/conf/create_mysql_table.sql)中的SQL命令，从而创建所需的MySQL库及表，默认创建的库名是`logi_kafka_manager`。 ``` # 示例： @@ -50,15 +56,38 @@ mysql -uXXXX -pXXX -h XXX.XXX.XXX.XXX -PXXXX < ./create_mysql_table.sql --- -## 4、启动 +## 5.修该配置 +请将`conf/application.yml.example` 文件复制一份出来命名为`application.yml` 放在同级目录:conf/application.yml ; +并且修改配置; 当然不修改的话 就会用默认的配置; +至少 mysql配置成自己的吧 -``` -# application.yml 是配置文件，最简单的是仅修改MySQL相关的配置即可启动 -nohup java -jar kafka-manager.jar --spring.config.location=./application.yml > /dev/null 2>&1 & -``` +## 6、启动/关闭 +解压包中有启动和关闭脚本 +`kafka-manager/bin/shutdown.sh` +`kafka-manager/bin/startup.sh` -### 5、使用 +执行 sh startup.sh 启动 +执行 sh shutdown.sh 关闭 + + + +### 6、使用 本地启动的话，访问`http://localhost:8080`，输入帐号及密码(默认`admin/admin`)进行登录。更多参考：[kafka-manager 用户使用手册](../user_guide/user_guide_cn.md) +### 7. 升级 + +如果是升级版本,请查看文件 [kafka-manager 升级手册](../../distribution/upgrade_config.md) + 在您下载的启动包(V2.5及其后)中也有记录,在 kafka-manager/upgrade_config.md 中 + + +### 8. 在IDE中启动 +> 如果想参与开发或者想在IDE中启动的话 +> 先执行 `mvn -Dmaven.test.skip=true clean install -U ` +> +> 然后这个时候可以选择去 [pom.xml](../../pom.xml) 中将`kafka-manager-console`模块注释掉; +> 注释是因为每次install的时候都会把前端文件`kafka-manager-console`重新打包进`kafka-manager-web` +> +> 完事之后,只需要直接用IDE启动运行`kafka-manager-web`模块中的 +> com.xiaojukeji.kafka.manager.web.MainApplication main方法就行了 \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/topic/MineTopicSummary.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/topic/MineTopicSummary.java index 45fffebd..7f02e51b 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/topic/MineTopicSummary.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/topic/MineTopicSummary.java @@ -25,6 +25,8 @@ public class MineTopicSummary { private Integer access; + private String description; + public Long getLogicalClusterId() { return logicalClusterId; } @@ -105,6 +107,14 @@ public class MineTopicSummary { this.access = access; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public String toString() { return "MineTopicSummary{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/order/OrderVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/order/OrderVO.java index 390d3ef4..e8b05779 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/order/OrderVO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/order/OrderVO.java @@ -27,8 +27,11 @@ public class OrderVO { @ApiModelProperty(value = "工单状态, 0:待审批, 1:通过, 2:拒绝, 3:取消") private Integer status; - @ApiModelProperty(value = "申请/审核时间") - private Date gmtTime; + @ApiModelProperty(value = "申请时间") + private Date gmtCreate; + + @ApiModelProperty(value = "审核时间") + private Date gmtHandle; public Long getId() { return id; @@ -70,12 +73,20 @@ public class OrderVO { this.status = status; } - public Date getGmtTime() { - return gmtTime; + public Date getGmtCreate() { + return gmtCreate; } - public void setGmtTime(Date gmtTime) { - this.gmtTime = gmtTime; + public void setGmtCreate(Date gmtCreate) { + this.gmtCreate = gmtCreate; + } + + public Date getGmtHandle() { + return gmtHandle; + } + + public void setGmtHandle(Date gmtHandle) { + this.gmtHandle = gmtHandle; } public String getApplicant() { @@ -95,7 +106,7 @@ public class OrderVO { ", applicant='" + applicant + '\'' + ", description='" + description + '\'' + ", status=" + status + - ", gmtTime=" + gmtTime + + ", gmtTime=" + gmtCreate + '}'; } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/topic/TopicMineVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/topic/TopicMineVO.java index 2cda46cb..2e4665a1 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/topic/TopicMineVO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/topic/TopicMineVO.java @@ -36,6 +36,9 @@ public class TopicMineVO { @ApiModelProperty(value = "状态, 0:无权限, 1:可消费 2:可发送 3:可消费发送 4:可管理") private Integer access; + @ApiModelProperty(value = "备注") + private String description; + public Long getClusterId() { return clusterId; } @@ -108,6 +111,14 @@ public class TopicMineVO { this.access = access; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public String toString() { return "TopicMineVO{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/BackoffUtils.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/BackoffUtils.java new file mode 100644 index 00000000..f4218020 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/BackoffUtils.java @@ -0,0 +1,20 @@ +package com.xiaojukeji.kafka.manager.common.utils; + +public class BackoffUtils { + private BackoffUtils() { + } + + public static void backoff(long timeUnitMs) { + if (timeUnitMs <= 0) { + return; + } + + try { + Thread.sleep(timeUnitMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + // ignore + } + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java index 5625b37f..56b3a1d5 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java @@ -1,5 +1,6 @@ package com.xiaojukeji.kafka.manager.common.utils.jmx; +import com.xiaojukeji.kafka.manager.common.utils.BackoffUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,18 +147,16 @@ public class JmxConnectorWrap { long now = System.currentTimeMillis(); while (true) { try { - if (System.currentTimeMillis() - now > 60000) { - break; - } int num = atomicInteger.get(); if (num <= 0) { - Thread.sleep(2); - continue; + BackoffUtils.backoff(2); } - if (atomicInteger.compareAndSet(num, num - 1)) { + + if (atomicInteger.compareAndSet(num, num - 1) || System.currentTimeMillis() - now > 6000) { break; } } catch (Exception e) { + // ignore } } } diff --git a/kafka-manager-console/package.json b/kafka-manager-console/package.json index 3be33c21..d768bc73 100644 --- a/kafka-manager-console/package.json +++ b/kafka-manager-console/package.json @@ -1,6 +1,6 @@ { "name": "logi-kafka", - "version": "2.4.3", + "version": "2.5.0", "description": "", "scripts": { "start": "webpack-dev-server", diff --git a/kafka-manager-console/src/assets/image/weChat.png b/kafka-manager-console/src/assets/image/weChat.png new file mode 100644 index 00000000..126f5816 Binary files /dev/null and b/kafka-manager-console/src/assets/image/weChat.png differ diff --git a/kafka-manager-console/src/assets/image/wechat.jpeg b/kafka-manager-console/src/assets/image/wechats.jpeg similarity index 100% rename from kafka-manager-console/src/assets/image/wechat.jpeg rename to kafka-manager-console/src/assets/image/wechats.jpeg diff --git a/kafka-manager-console/src/component/chart/date-picker-chart.tsx b/kafka-manager-console/src/component/chart/date-picker-chart.tsx index 0e0d6d6a..cda4a6d0 100644 --- a/kafka-manager-console/src/component/chart/date-picker-chart.tsx +++ b/kafka-manager-console/src/component/chart/date-picker-chart.tsx @@ -60,6 +60,22 @@ export class ChartWithDatePicker extends React.Component { public changeChartOptions(options: any) { const noData = options.series.length ? false : true; this.setState({ noData }); + options.tooltip.formatter = (params: any) => { + var res = + "

" + + params[0].data.time + + "

"; + for (var i = 0; i < params.length; i++) { + res += `
+ + ${params[i].seriesName} + ${params[i].data[params[i].seriesName]} +
`; + } + return res; + } this.chart.setOption(options, true); } @@ -79,7 +95,7 @@ export class ChartWithDatePicker extends React.Component { public render() { const { customerNode } = this.props; return ( -
+
{customerNode} diff --git a/kafka-manager-console/src/container/header/index.tsx b/kafka-manager-console/src/container/header/index.tsx index e12e397c..c668335a 100644 --- a/kafka-manager-console/src/container/header/index.tsx +++ b/kafka-manager-console/src/container/header/index.tsx @@ -7,7 +7,7 @@ import { urlPrefix } from 'constants/left-menu'; import { region, IRegionIdcs } from 'store/region'; import logoUrl from '../../assets/image/kafka-logo.png'; import userIcon from '../../assets/image/normal.png'; -import weChat from '../../assets/image/wechat.jpeg'; +import weChat from '../../assets/image/wechat.png'; import { users } from 'store/users'; import { observer } from 'mobx-react'; import { Link } from 'react-router-dom'; @@ -60,8 +60,8 @@ export const Header = observer((props: IHeader) => { }); }; const content = ( -
- +
+
); const helpCenter = ( @@ -144,7 +144,7 @@ export const Header = observer((props: IHeader) => {
- Kafka Manager + LogiKM v2.4.2 {/* 添加版本超链接 */}
diff --git a/kafka-manager-console/src/container/user-center/order-list.tsx b/kafka-manager-console/src/container/user-center/order-list.tsx index 6c81b0ec..5ed5b961 100644 --- a/kafka-manager-console/src/container/user-center/order-list.tsx +++ b/kafka-manager-console/src/container/user-center/order-list.tsx @@ -115,11 +115,19 @@ export class OrderList extends SearchAndFilterContainer { status, { title: '申请时间', - dataIndex: 'gmtTime', - key: 'gmtTime', - sorter: (a: IBaseOrder, b: IBaseOrder) => b.gmtTime - a.gmtTime, - render: (t: number) => moment(t).format(timeFormat), - }, { + dataIndex: 'gmtCreate', + key: 'gmtCreate', + sorter: (a: IBaseOrder, b: IBaseOrder) => b.gmtCreate - a.gmtCreate, + render: (t: number) => t ? moment(t).format(timeFormat) : '-', + }, + { + title: '审批时间', + dataIndex: 'gmtHandle', + key: 'gmtHandle', + sorter: (a: IBaseOrder, b: IBaseOrder) => b.gmtHandle - a.gmtHandle, + render: (t: number) => t ? moment(t).format(timeFormat) : '-', + }, + { title: '操作', key: 'operation', dataIndex: 'operation', diff --git a/kafka-manager-console/src/routers/index.htm b/kafka-manager-console/src/routers/index.htm index b8d8454f..7cb7a0fd 100644 --- a/kafka-manager-console/src/routers/index.htm +++ b/kafka-manager-console/src/routers/index.htm @@ -1,12 +1,15 @@ + - KafkaManager + LogiKM +
+ \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicExpiredService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicExpiredService.java index 273b62c6..9ab963aa 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicExpiredService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicExpiredService.java @@ -13,4 +13,12 @@ public interface TopicExpiredService { List getExpiredTopicDataList(String username); ResultStatus retainExpiredTopic(Long physicalClusterId, String topicName, Integer retainDays); + + /** + * 通过topictopic名称删除 + * @param clusterId 集群id + * @param topicName topic名称 + * @return int + */ + int deleteByTopicName(Long clusterId, String topicName); } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java index 8a0028c7..594f1aa1 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java @@ -43,6 +43,9 @@ public class AdminServiceImpl implements AdminService { @Autowired private TopicManagerService topicManagerService; + @Autowired + private TopicExpiredService topicExpiredService; + @Autowired private TopicService topicService; @@ -143,6 +146,7 @@ public class AdminServiceImpl implements AdminService { // 3. 数据库中删除topic topicManagerService.deleteByTopicName(clusterDO.getId(), topicName); + topicExpiredService.deleteByTopicName(clusterDO.getId(), topicName); // 4. 数据库中删除authority authorityService.deleteAuthorityByTopic(clusterDO.getId(), topicName); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java index ea9d22da..153576c4 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java @@ -19,6 +19,8 @@ import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -210,7 +212,7 @@ public class ClusterServiceImpl implements ClusterService { ZooKeeper zk = null; try { - zk = new ZooKeeper(zookeeper, 1000, null); + zk = new ZooKeeper(zookeeper, 1000, watchedEvent -> LOGGER.info(" receive event : " + watchedEvent.getType().name())); for (int i = 0; i < 15; ++i) { if (zk.getState().isConnected()) { // 只有状态是connected的时候，才表示地址是合法的 diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicExpiredServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicExpiredServiceImpl.java index c51e1dcb..d310af1a 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicExpiredServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicExpiredServiceImpl.java @@ -75,4 +75,14 @@ public class TopicExpiredServiceImpl implements TopicExpiredService { } return ResultStatus.MYSQL_ERROR; } + + @Override + public int deleteByTopicName(Long clusterId, String topicName) { + try { + return topicExpiredDao.deleteByName(clusterId, topicName); + } catch (Exception e) { + LOGGER.error("delete topic failed, clusterId:{} topicName:{}", clusterId, topicName, e); + } + return 0; + } } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java index 4a8f501f..a25115ef 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java @@ -210,7 +210,7 @@ public class TopicManagerServiceImpl implements TopicManagerService { } } - // 增加流量信息 + // 增加流量和描述信息 Map> metricMap = KafkaMetricsCache.getAllTopicMetricsFromCache(); for (MineTopicSummary mineTopicSummary : summaryList) { TopicMetrics topicMetrics = getTopicMetricsFromCacheOrJmx( @@ -219,6 +219,10 @@ public class TopicManagerServiceImpl implements TopicManagerService { metricMap); mineTopicSummary.setBytesIn(topicMetrics.getSpecifiedMetrics("BytesInPerSecOneMinuteRate")); mineTopicSummary.setBytesOut(topicMetrics.getSpecifiedMetrics("BytesOutPerSecOneMinuteRate")); + + // 增加topic描述信息 + TopicDO topicDO = topicDao.getByTopicName(mineTopicSummary.getPhysicalClusterId(), mineTopicSummary.getTopicName()); + mineTopicSummary.setDescription(topicDO.getDescription()); } return summaryList; } diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicExpiredDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicExpiredDao.java index 18698941..ea189eb4 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicExpiredDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicExpiredDao.java @@ -17,4 +17,6 @@ public interface TopicExpiredDao { int replace(TopicExpiredDO expiredDO); TopicExpiredDO getByTopic(Long clusterId, String topicName); + + int deleteByName(Long clusterId, String topicName); } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicExpiredDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicExpiredDaoImpl.java index 51853db7..936d4931 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicExpiredDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicExpiredDaoImpl.java @@ -50,4 +50,12 @@ public class TopicExpiredDaoImpl implements TopicExpiredDao { params.put("topicName", topicName); return sqlSession.selectOne("TopicExpiredDao.getByTopic", params); } + + @Override + public int deleteByName(Long clusterId, String topicName) { + Map params = new HashMap<>(2); + params.put("clusterId", clusterId); + params.put("topicName", topicName); + return sqlSession.delete("TopicExpiredDao.deleteByName", params); + } } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/resources/mapper/TopicExpiredDao.xml b/kafka-manager-dao/src/main/resources/mapper/TopicExpiredDao.xml index 39ebf8ca..1da6753a 100644 --- a/kafka-manager-dao/src/main/resources/mapper/TopicExpiredDao.xml +++ b/kafka-manager-dao/src/main/resources/mapper/TopicExpiredDao.xml @@ -36,4 +36,8 @@ + + + DELETE FROM topic_expired WHERE cluster_id=#{clusterId} AND topic_name=#{topicName} + diff --git a/kafka-manager-dao/src/main/resources/mapper/TopicMetricsDao.xml b/kafka-manager-dao/src/main/resources/mapper/TopicMetricsDao.xml index baa6f4b0..53e13b2d 100644 --- a/kafka-manager-dao/src/main/resources/mapper/TopicMetricsDao.xml +++ b/kafka-manager-dao/src/main/resources/mapper/TopicMetricsDao.xml @@ -25,6 +25,7 @@ WHERE cluster_id = #{clusterId} AND topic_name = #{topicName} AND gmt_create BETWEEN #{startTime} AND #{endTime} + ORDER BY gmt_create ]]> @@ -32,6 +33,7 @@ diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCommunityTopicMetrics.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCommunityTopicMetrics.java index 07a137e6..a6757310 100644 --- a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCommunityTopicMetrics.java +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCommunityTopicMetrics.java @@ -30,16 +30,23 @@ public class CollectAndPublishCommunityTopicMetrics extends AbstractScheduledTas @Override protected List listAllTasks() { + // 获取需要进行指标采集的集群列表，这些集群将会被拆分到多台KM中进行执行。 return clusterService.list(); } @Override public void processTask(ClusterDO clusterDO) { + // 这里需要实现对clusterDO这个集群进行Topic指标采集的代码逻辑 + + // 进行Topic指标获取 List metricsList = getTopicMetrics(clusterDO.getId()); + + // 获取到Topic流量指标之后，发布一个事件， SpringTool.publish(new TopicMetricsCollectedEvent(this, clusterDO.getId(), metricsList)); } private List getTopicMetrics(Long clusterId) { + // 具体获取Topic流量指标的入口代码 List metricsList = jmxService.getTopicMetrics(clusterId, KafkaMetricsCollections.TOPIC_METRICS_TO_DB, true); if (ValidateUtils.isEmptyList(metricsList)) { diff --git a/kafka-manager-web/pom.xml b/kafka-manager-web/pom.xml index 17504ca7..a959f958 100644 --- a/kafka-manager-web/pom.xml +++ b/kafka-manager-web/pom.xml @@ -19,7 +19,7 @@ 2.1.1.RELEASE 5.1.3.RELEASE false - 8.5.66 + 8.5.72 @@ -109,8 +109,10 @@ + kafka-manager + org.springframework.boot spring-boot-maven-plugin ${springframework.boot.version} @@ -121,6 +123,7 @@ + diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/OrderConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/OrderConverter.java index bbe8c656..ab6c0ba6 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/OrderConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/OrderConverter.java @@ -1,15 +1,16 @@ package com.xiaojukeji.kafka.manager.web.converters; -import com.xiaojukeji.kafka.manager.common.entity.ao.account.Account; import com.xiaojukeji.kafka.manager.bpm.common.OrderResult; +import com.xiaojukeji.kafka.manager.bpm.common.OrderStatusEnum; import com.xiaojukeji.kafka.manager.bpm.common.entry.BaseOrderDetailData; +import com.xiaojukeji.kafka.manager.common.entity.ao.account.Account; +import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO; import com.xiaojukeji.kafka.manager.common.entity.vo.common.AccountVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.order.OrderResultVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.order.OrderVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.order.detail.OrderDetailBaseVO; import com.xiaojukeji.kafka.manager.common.utils.CopyUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; -import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO; import java.util.ArrayList; import java.util.Collections; @@ -41,7 +42,9 @@ public class OrderConverter { } OrderVO orderVO = new OrderVO(); CopyUtils.copyProperties(orderVO, orderDO); - orderVO.setGmtTime(orderDO.getGmtCreate()); + if (OrderStatusEnum.WAIT_DEAL.getCode().equals(orderDO.getStatus())) { + orderVO.setGmtHandle(null); + } return orderVO; } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicMineConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicMineConverter.java index 97b8f04a..e21c41da 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicMineConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicMineConverter.java @@ -29,6 +29,7 @@ public class TopicMineConverter { vo.setClusterName(data.getLogicalClusterName()); vo.setBytesIn(data.getBytesIn()); vo.setBytesOut(data.getBytesOut()); + vo.setDescription(data.getDescription()); voList.add(vo); } return voList; diff --git a/kafka-manager-web/src/main/resources/application.yml b/kafka-manager-web/src/main/resources/application.yml index 4463d746..a4648a46 100644 --- a/kafka-manager-web/src/main/resources/application.yml +++ b/kafka-manager-web/src/main/resources/application.yml @@ -9,6 +9,8 @@ server: spring: application: name: kafkamanager + profiles: + active: dev datasource: kafka-manager: jdbc-url: jdbc:mysql://localhost:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8 @@ -18,8 +20,6 @@ spring: main: allow-bean-definition-overriding: true - profiles: - active: dev servlet: multipart: max-file-size: 100MB diff --git a/pom.xml b/pom.xml index a7c70e54..235e8165 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ - 2.4.2-SNAPSHOT + 2.4.3-SNAPSHOT 2.7.0 1.5.13 @@ -26,11 +26,15 @@ 1.8 UTF-8 UTF-8 - 8.5.66 + 8.5.72 + 3.0.0 + + kafka-manager-common kafka-manager-dao kafka-manager-core @@ -42,6 +46,7 @@ kafka-manager-extends/kafka-manager-openapi kafka-manager-task kafka-manager-web + distribution @@ -231,4 +236,16 @@ + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + \ No newline at end of file