diff --git a/Releases_Notes.md b/Releases_Notes.md new file mode 100644 index 00000000..46b5753e --- /dev/null +++ b/Releases_Notes.md @@ -0,0 +1,97 @@ + +--- + +![kafka-manager-logo](./docs/assets/images/common/logo_name.png) + +**一站式`Apache Kafka`集群指标监控与运维管控平台** + +--- + +## v2.3.0 + +版本上线时间：2021-02-08 + + +### 能力提升 + +- 新增支持docker化部署 +- 可指定Broker作为候选controller +- 可新增并管理网关配置 +- 可获取消费组状态 +- 增加集群的JMX认证 + +### 体验优化 + +- 优化编辑用户角色、修改密码的流程 +- 新增consumerID的搜索功能 +- 优化“Topic连接信息”、“消费组重置消费偏移”、“修改Topic保存时间”的文案提示 +- 在相应位置增加《资源申请文档》链接 + +### bug修复 + +- 修复Broker监控图表时间轴展示错误的问题 +- 修复创建夜莺监控告警规则时，使用的告警周期的单位不正确的问题 + + + +## v2.2.0 + +版本上线时间：2021-01-25 + + + +### 能力提升 + +- 优化工单批量操作流程 +- 增加获取Topic75分位/99分位的实时耗时数据 +- 增加定时任务，可将无主未落DB的Topic定期写入DB + +### 体验优化 + +- 在相应位置增加《集群接入文档》链接 +- 优化物理集群、逻辑集群含义 +- 在Topic详情页、Topic扩分区操作弹窗增加展示Topic所属Region的信息 +- 优化Topic审批时，Topic数据保存时间的配置流程 +- 优化Topic/应用申请、审批时的错误提示文案 +- 优化Topic数据采样的操作项文案 +- 优化运维人员删除Topic时的提示文案 +- 优化运维人员删除Region的删除逻辑与提示文案 +- 优化运维人员删除逻辑集群的提示文案 +- 优化上传集群配置文件时的文件类型限制条件 + +### bug修复 + +- 修复填写应用名称时校验特殊字符出错的问题 +- 修复普通用户越权访问应用详情的问题 +- 修复由于Kafka版本升级，导致的数据压缩格式无法获取的问题 +- 修复删除逻辑集群或Topic之后，界面依旧展示的问题 +- 修复进行Leader rebalance操作时执行结果重复提示的问题 + + +## v2.1.0 + +版本上线时间：2020-12-19 + + + +### 体验优化 + +- 优化页面加载时的背景样式 +- 优化普通用户申请Topic权限的流程 +- 优化Topic申请配额、申请分区的权限限制 +- 优化取消Topic权限的文案提示 +- 优化申请配额表单的表单项名称 +- 优化重置消费偏移的操作流程 +- 优化创建Topic迁移任务的表单内容 +- 优化Topic扩分区操作的弹窗界面样式 +- 优化集群Broker监控可视化图表样式 +- 优化创建逻辑集群的表单内容 +- 优化集群安全协议的提示文案 + +### bug修复 + +- 修复偶发性重置消费偏移失败的问题 + + + + diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 8ab9781a..f62ba59f 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -18,6 +18,7 @@ - 6、如何使用`MySQL 8`？ - 7、`Jmx`连接失败如何解决？ - 8、`topic biz data not exist`错误及处理方式 +- 9、进程启动后，如何查看API文档 --- @@ -76,7 +77,7 @@ - 3、数据库时区问题。 -检查MySQL的topic表，查看是否有数据，如果有数据，那么再检查设置的时区是否正确。 +检查MySQL的topic_metrics表，查看是否有数据，如果有数据，那么再检查设置的时区是否正确。 --- @@ -109,3 +110,7 @@ 可以在`运维管控->集群列表->Topic信息`下面，编辑申请权限的Topic，为Topic选择一个应用即可。 以上仅仅只是针对单个Topic的场景，如果你有非常多的Topic需要进行初始化的，那么此时可以在配置管理中增加一个配置，来定时的对无主的Topic进行同步，具体见：[动态配置管理 - 1、Topic定时同步任务](../dev_guide/dynamic_config_manager.md) + +### 9、进程启动后，如何查看API文档 + +- 滴滴Logi-KafkaManager采用Swagger-API工具记录API文档。Swagger-API地址： [http://IP:PORT/swagger-ui.html#/](http://IP:PORT/swagger-ui.html#/) diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java index 94acb56d..8f0f229b 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java @@ -106,6 +106,7 @@ public enum ResultStatus { STORAGE_UPLOAD_FILE_FAILED(8050, "upload file failed"), STORAGE_FILE_TYPE_NOT_SUPPORT(8051, "File type not support"), STORAGE_DOWNLOAD_FILE_FAILED(8052, "download file failed"), + LDAP_AUTHENTICATION_FAILED(8053, "LDAP authentication failed"), ; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/ResultStatus.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/ResultStatus.java new file mode 100644 index 00000000..663dd0a6 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/ResultStatus.java @@ -0,0 +1,136 @@ +package com.xiaojukeji.kafka.manager.common.entity; + +import com.xiaojukeji.kafka.manager.common.constant.Constant; + +/** + * 返回状态 + * @author zengqiao + * @date 20/4/16 + */ +public enum ResultStatus { + GATEWAY_INVALID_REQUEST(-1, "invalid request"), + + SUCCESS(Constant.SUCCESS, "success"), + + FAIL(1, "操作失败"), + + /** + * 操作错误[1000, 2000) + * ------------------------------------------------------------------------------------------ + */ + OPERATION_FAILED(1401, "operation failed"), + OPERATION_FORBIDDEN(1402, "operation forbidden"), + API_CALL_EXCEED_LIMIT(1403, "api call exceed limit"), + USER_WITHOUT_AUTHORITY(1404, "user without authority"), + CHANGE_ZOOKEEPER_FORBIDDEN(1405, "change zookeeper forbidden"), + + + TOPIC_OPERATION_PARAM_NULL_POINTER(1450, "参数错误"), + TOPIC_OPERATION_PARTITION_NUM_ILLEGAL(1451, "分区数错误"), + TOPIC_OPERATION_BROKER_NUM_NOT_ENOUGH(1452, "Broker数不足错误"), + TOPIC_OPERATION_TOPIC_NAME_ILLEGAL(1453, "Topic名称非法"), + TOPIC_OPERATION_TOPIC_EXISTED(1454, "Topic已存在"), + TOPIC_OPERATION_UNKNOWN_TOPIC_PARTITION(1455, "Topic未知"), + TOPIC_OPERATION_TOPIC_CONFIG_ILLEGAL(1456, "Topic配置错误"), + TOPIC_OPERATION_TOPIC_IN_DELETING(1457, "Topic正在删除"), + TOPIC_OPERATION_UNKNOWN_ERROR(1458, "未知错误"), + + /** + * 参数错误[2000, 3000) + * ------------------------------------------------------------------------------------------ + */ + PARAM_ILLEGAL(2000, "param illegal"), + CG_LOCATION_ILLEGAL(2001, "consumer group location illegal"), + ORDER_ALREADY_HANDLED(2002, "order already handled"), + APP_ID_OR_PASSWORD_ILLEGAL(2003, "app or password illegal"), + SYSTEM_CODE_ILLEGAL(2004, "system code illegal"), + CLUSTER_TASK_HOST_LIST_ILLEGAL(2005, "主机列表错误，请检查主机列表"), + JSON_PARSER_ERROR(2006, "json parser error"), + + BROKER_NUM_NOT_ENOUGH(2050, "broker not enough"), + CONTROLLER_NOT_ALIVE(2051, "controller not alive"), + CLUSTER_METADATA_ERROR(2052, "cluster metadata error"), + TOPIC_CONFIG_ERROR(2053, "topic config error"), + + /** + * 参数错误 - 资源检查错误 + * 因为外部系统的问题, 操作时引起的错误, [7000, 8000) + * ------------------------------------------------------------------------------------------ + */ + RESOURCE_NOT_EXIST(7100, "资源不存在"), + CLUSTER_NOT_EXIST(7101, "cluster not exist"), + BROKER_NOT_EXIST(7102, "broker not exist"), + TOPIC_NOT_EXIST(7103, "topic not exist"), + PARTITION_NOT_EXIST(7104, "partition not exist"), + ACCOUNT_NOT_EXIST(7105, "account not exist"), + APP_NOT_EXIST(7106, "app not exist"), + ORDER_NOT_EXIST(7107, "order not exist"), + CONFIG_NOT_EXIST(7108, "config not exist"), + IDC_NOT_EXIST(7109, "idc not exist"), + TASK_NOT_EXIST(7110, "task not exist"), + AUTHORITY_NOT_EXIST(7111, "authority not exist"), + MONITOR_NOT_EXIST(7112, "monitor not exist"), + QUOTA_NOT_EXIST(7113, "quota not exist, please check clusterId, topicName and appId"), + CONSUMER_GROUP_NOT_EXIST(7114, "consumerGroup not exist"), + TOPIC_BIZ_DATA_NOT_EXIST(7115, "topic biz data not exist, please sync topic to db"), + + + // 资源已存在 + RESOURCE_ALREADY_EXISTED(7200, "资源已经存在"), + TOPIC_ALREADY_EXIST(7201, "topic already existed"), + + // 资源重名 + RESOURCE_NAME_DUPLICATED(7300, "资源名称重复"), + + // 资源已被使用 + RESOURCE_ALREADY_USED(7400, "资源早已被使用"), + + + /** + * 因为外部系统的问题, 操作时引起的错误, [8000, 9000) + * ------------------------------------------------------------------------------------------ + */ + MYSQL_ERROR(8010, "operate database failed"), + + ZOOKEEPER_CONNECT_FAILED(8020, "zookeeper connect failed"), + ZOOKEEPER_READ_FAILED(8021, "zookeeper read failed"), + ZOOKEEPER_WRITE_FAILED(8022, "zookeeper write failed"), + ZOOKEEPER_DELETE_FAILED(8023, "zookeeper delete failed"), + + // 调用集群任务里面的agent失败 + CALL_CLUSTER_TASK_AGENT_FAILED(8030, " call cluster task agent failed"), + + // 调用监控系统失败 + CALL_MONITOR_SYSTEM_ERROR(8040, " call monitor-system failed"), + + // 存储相关的调用失败 + STORAGE_UPLOAD_FILE_FAILED(8050, "upload file failed"), + STORAGE_FILE_TYPE_NOT_SUPPORT(8051, "File type not support"), + STORAGE_DOWNLOAD_FILE_FAILED(8052, "download file failed"), + + ; + + private int code; + private String message; + + ResultStatus(int code, String message) { + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/ResultStatus.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/ResultStatus.java new file mode 100644 index 00000000..94acb56d --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/ResultStatus.java @@ -0,0 +1,135 @@ +package com.xiaojukeji.kafka.manager.common.entity; + +import com.xiaojukeji.kafka.manager.common.constant.Constant; + +/** + * 返回状态 + * @author zengqiao + * @date 20/4/16 + */ +public enum ResultStatus { + GATEWAY_INVALID_REQUEST(-1, "invalid request"), + + SUCCESS(Constant.SUCCESS, "success"), + + FAIL(1, "操作失败"), + + /** + * 操作错误[1000, 2000) + * ------------------------------------------------------------------------------------------ + */ + OPERATION_FAILED(1401, "operation failed"), + OPERATION_FORBIDDEN(1402, "operation forbidden"), + API_CALL_EXCEED_LIMIT(1403, "api call exceed limit"), + USER_WITHOUT_AUTHORITY(1404, "user without authority"), + CHANGE_ZOOKEEPER_FORBIDDEN(1405, "change zookeeper forbidden"), + + + TOPIC_OPERATION_PARAM_NULL_POINTER(1450, "参数错误"), + TOPIC_OPERATION_PARTITION_NUM_ILLEGAL(1451, "分区数错误"), + TOPIC_OPERATION_BROKER_NUM_NOT_ENOUGH(1452, "Broker数不足错误"), + TOPIC_OPERATION_TOPIC_NAME_ILLEGAL(1453, "Topic名称非法"), + TOPIC_OPERATION_TOPIC_EXISTED(1454, "Topic已存在"), + TOPIC_OPERATION_UNKNOWN_TOPIC_PARTITION(1455, "Topic未知"), + TOPIC_OPERATION_TOPIC_CONFIG_ILLEGAL(1456, "Topic配置错误"), + TOPIC_OPERATION_TOPIC_IN_DELETING(1457, "Topic正在删除"), + TOPIC_OPERATION_UNKNOWN_ERROR(1458, "未知错误"), + + /** + * 参数错误[2000, 3000) + * ------------------------------------------------------------------------------------------ + */ + PARAM_ILLEGAL(2000, "param illegal"), + CG_LOCATION_ILLEGAL(2001, "consumer group location illegal"), + ORDER_ALREADY_HANDLED(2002, "order already handled"), + APP_ID_OR_PASSWORD_ILLEGAL(2003, "app or password illegal"), + SYSTEM_CODE_ILLEGAL(2004, "system code illegal"), + CLUSTER_TASK_HOST_LIST_ILLEGAL(2005, "主机列表错误，请检查主机列表"), + JSON_PARSER_ERROR(2006, "json parser error"), + + BROKER_NUM_NOT_ENOUGH(2050, "broker not enough"), + CONTROLLER_NOT_ALIVE(2051, "controller not alive"), + CLUSTER_METADATA_ERROR(2052, "cluster metadata error"), + TOPIC_CONFIG_ERROR(2053, "topic config error"), + + /** + * 参数错误 - 资源检查错误 + * 因为外部系统的问题, 操作时引起的错误, [7000, 8000) + * ------------------------------------------------------------------------------------------ + */ + RESOURCE_NOT_EXIST(7100, "资源不存在"), + CLUSTER_NOT_EXIST(7101, "cluster not exist"), + BROKER_NOT_EXIST(7102, "broker not exist"), + TOPIC_NOT_EXIST(7103, "topic not exist"), + PARTITION_NOT_EXIST(7104, "partition not exist"), + ACCOUNT_NOT_EXIST(7105, "account not exist"), + APP_NOT_EXIST(7106, "app not exist"), + ORDER_NOT_EXIST(7107, "order not exist"), + CONFIG_NOT_EXIST(7108, "config not exist"), + IDC_NOT_EXIST(7109, "idc not exist"), + TASK_NOT_EXIST(7110, "task not exist"), + AUTHORITY_NOT_EXIST(7111, "authority not exist"), + MONITOR_NOT_EXIST(7112, "monitor not exist"), + QUOTA_NOT_EXIST(7113, "quota not exist, please check clusterId, topicName and appId"), + CONSUMER_GROUP_NOT_EXIST(7114, "consumerGroup not exist"), + TOPIC_BIZ_DATA_NOT_EXIST(7115, "topic biz data not exist, please sync topic to db"), + + // 资源已存在 + RESOURCE_ALREADY_EXISTED(7200, "资源已经存在"), + TOPIC_ALREADY_EXIST(7201, "topic already existed"), + + // 资源重名 + RESOURCE_NAME_DUPLICATED(7300, "资源名称重复"), + + // 资源已被使用 + RESOURCE_ALREADY_USED(7400, "资源早已被使用"), + + + /** + * 因为外部系统的问题, 操作时引起的错误, [8000, 9000) + * ------------------------------------------------------------------------------------------ + */ + MYSQL_ERROR(8010, "operate database failed"), + + ZOOKEEPER_CONNECT_FAILED(8020, "zookeeper connect failed"), + ZOOKEEPER_READ_FAILED(8021, "zookeeper read failed"), + ZOOKEEPER_WRITE_FAILED(8022, "zookeeper write failed"), + ZOOKEEPER_DELETE_FAILED(8023, "zookeeper delete failed"), + + // 调用集群任务里面的agent失败 + CALL_CLUSTER_TASK_AGENT_FAILED(8030, " call cluster task agent failed"), + + // 调用监控系统失败 + CALL_MONITOR_SYSTEM_ERROR(8040, " call monitor-system failed"), + + // 存储相关的调用失败 + STORAGE_UPLOAD_FILE_FAILED(8050, "upload file failed"), + STORAGE_FILE_TYPE_NOT_SUPPORT(8051, "File type not support"), + STORAGE_DOWNLOAD_FILE_FAILED(8052, "download file failed"), + + ; + + private int code; + private String message; + + ResultStatus(int code, String message) { + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java index 2419901a..eff3bc25 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java @@ -1,5 +1,6 @@ package com.xiaojukeji.kafka.manager.common.utils.ldap; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -25,6 +26,9 @@ public class LDAPAuthentication { @Value(value = "${ldap.factory}") private String ldapFactory; + @Value(value = "${ldap.filter}") + private String ldapfilter; + @Value(value = "${ldap.auth-user-registration-role}") private String authUserRegistrationRole; @@ -61,7 +65,9 @@ public class LDAPAuthentication { try { SearchControls constraints = new SearchControls(); constraints.setSearchScope(SearchControls.SUBTREE_SCOPE); - NamingEnumeration en = ctx.search("", "account=" + account, constraints); + String filter = "(&(objectClass=*)("+ldapfilter+"=" + account + "))"; + + NamingEnumeration en = ctx.search("", filter, constraints); if (en == null || !en.hasMoreElements()) { return ""; } @@ -95,7 +101,9 @@ public class LDAPAuthentication { try { String userDN = getUserDN(account,ctx); - + if(ValidateUtils.isBlank(userDN)){ + return valide; + } ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN); ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password); ctx.reconnect(null); diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java index 3aa0e703..c67cca08 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java @@ -5,6 +5,7 @@ import com.xiaojukeji.kafka.manager.account.component.AbstractSingleSignOn; import com.xiaojukeji.kafka.manager.common.bizenum.AccountRoleEnum; import com.xiaojukeji.kafka.manager.common.constant.LoginConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.LoginDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.AccountDO; import com.xiaojukeji.kafka.manager.common.utils.EncryptUtil; @@ -44,7 +45,7 @@ public class BaseSessionSignOn extends AbstractSingleSignOn { @Override public Result loginAndGetLdap(HttpServletRequest request, HttpServletResponse response, LoginDTO dto) { if (ValidateUtils.isBlank(dto.getUsername()) || ValidateUtils.isNull(dto.getPassword())) { - return null; + return Result.buildFailure("Missing parameters"); } Result accountResult = accountService.getAccountDO(dto.getUsername()); @@ -54,7 +55,7 @@ public class BaseSessionSignOn extends AbstractSingleSignOn { if(ldapEnabled){ //去LDAP验证账密 if(!ldapAuthentication.authenricate(dto.getUsername(),dto.getPassword())){ - return null; + return Result.buildFrom(ResultStatus.LDAP_AUTHENTICATION_FAILED); } if((ValidateUtils.isNull(accountResult) || ValidateUtils.isNull(accountResult.getData())) && authUserRegistration){ diff --git a/kafka-manager-web/src/main/resources/application.yml b/kafka-manager-web/src/main/resources/application.yml index 008cc048..89fca91c 100644 --- a/kafka-manager-web/src/main/resources/application.yml +++ b/kafka-manager-web/src/main/resources/application.yml @@ -89,6 +89,7 @@ ldap: url: ldap://127.0.0.1:389/ basedn: dc=tsign,dc=cn factory: com.sun.jndi.ldap.LdapCtxFactory + filter: sAMAccountName security: authentication: simple principal: cn=admin,dc=tsign,dc=cn