diff --git a/README.md b/README.md index d0887cac..aaa7e1d8 100644 --- a/README.md +++ b/README.md @@ -53,11 +53,20 @@ ## 2 相关文档 +### 2.1 产品文档 - [滴滴Logi-KafkaManager 安装手册](docs/install_guide/install_guide_cn.md) - [滴滴Logi-KafkaManager 接入集群](docs/user_guide/add_cluster/add_cluster.md) - [滴滴Logi-KafkaManager 用户使用手册](docs/user_guide/user_guide_cn.md) - [滴滴Logi-KafkaManager FAQ](docs/user_guide/faq.md) +### 2.2 社区文章 +- [滴滴云官网产品介绍](https://www.didiyun.com/production/logi-KafkaManager.html) +- [7年沉淀之作--滴滴Logi日志服务套件](https://mp.weixin.qq.com/s/-KQp-Qo3WKEOc9wIR2iFnw) +- [滴滴Logi-KafkaManager 一站式Kafka监控与管控平台](https://mp.weixin.qq.com/s/9qSZIkqCnU6u9nLMvOOjIQ) +- [滴滴Logi-KafkaManager 开源之路](https://xie.infoq.cn/article/0223091a99e697412073c0d64) +- [滴滴Logi-KafkaManager 系列视频教程](https://mp.weixin.qq.com/s/9X7gH0tptHPtfjPPSdGO8g) +- [kafka实践（十五）：滴滴开源Kafka管控平台 Logi-KafkaManager研究--A叶子叶来](https://blog.csdn.net/yezonggang/article/details/113106244) + ## 3 滴滴Logi开源用户钉钉交流群 ![dingding_group](./docs/assets/images/common/dingding_group.jpg) diff --git a/build.sh b/build.sh index 03b1087e..f3ea8642 100644 --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ cd $workspace ## constant OUTPUT_DIR=./output -KM_VERSION=2.2.0 +KM_VERSION=2.3.0 APP_NAME=kafka-manager APP_DIR=${APP_NAME}-${KM_VERSION} diff --git a/docs/dev_guide/upgrade_manual/logi-km-v2.3.0.md b/docs/dev_guide/upgrade_manual/logi-km-v2.3.0.md new file mode 100644 index 00000000..3a4196f8 --- /dev/null +++ b/docs/dev_guide/upgrade_manual/logi-km-v2.3.0.md @@ -0,0 +1,17 @@ + +--- + +![kafka-manager-logo](../../assets/images/common/logo_name.png) + +**一站式`Apache Kafka`集群指标监控与运维管控平台** + +--- + +# 升级至`2.3.0`版本 + +`2.3.0`版本在`gateway_config`表增加了一个描述说明的字段，因此需要执行下面的sql进行字段的增加。 + +```sql +ALTER TABLE `gateway_config` +ADD COLUMN `description` TEXT NULL COMMENT '描述信息' AFTER `version`; +``` diff --git a/docs/install_guide/create_mysql_table.sql b/docs/install_guide/create_mysql_table.sql index 2a015de1..065532eb 100644 --- a/docs/install_guide/create_mysql_table.sql +++ b/docs/install_guide/create_mysql_table.sql @@ -203,7 +203,8 @@ CREATE TABLE `gateway_config` ( `type` varchar(128) NOT NULL DEFAULT '' COMMENT '配置类型', `name` varchar(128) NOT NULL DEFAULT '' COMMENT '配置名称', `value` text COMMENT '配置值', - `version` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '版本信息', + `version` bigint(20) unsigned NOT NULL DEFAULT '1' COMMENT '版本信息', + `description` text COMMENT '描述信息', `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', PRIMARY KEY (`id`), diff --git a/docs/user_guide/add_cluster/add_cluster.md b/docs/user_guide/add_cluster/add_cluster.md index 14c1d907..1774a9be 100644 --- a/docs/user_guide/add_cluster/add_cluster.md +++ b/docs/user_guide/add_cluster/add_cluster.md @@ -5,16 +5,26 @@ **一站式`Apache Kafka`集群指标监控与运维管控平台** + --- # 集群接入 -集群的接入总共需要三个步骤，分别是： -1. 接入物理集群 -2. 创建Region -3. 创建逻辑集群 +## 主要概念讲解 +面对大规模集群、业务场景复杂的情况，引入Region、逻辑集群的概念 +- Region：划分部分Broker作为一个 Region，用Region定义资源划分的单位，提高扩展性和隔离性。如果部分Topic异常也不会影响大面积的Broker +- 逻辑集群：逻辑集群由部分Region组成，便于对大规模集群按照业务划分、保障能力进行管理 +![op_cluster_arch](assets/op_cluster_arch.png) -备注：接入集群需要2、3两步是因为普通用户的视角下，看到的都是逻辑集群，如果没有2、3两步，那么普通用户看不到任何信息。 +集群的接入总共需要三个步骤，分别是： +1. 接入物理集群：填写机器地址、安全协议等配置信息，接入真实的物理集群 +2. 创建Region：将部分Broker划分为一个Region +3. 创建逻辑集群：逻辑集群由部分Region组成，可根据业务划分、保障等级来创建相应的逻辑集群 + +![op_cluster_flow](assets/op_cluster_flow.png) + + +**备注：接入集群需要2、3两步是因为普通用户的视角下，看到的都是逻辑集群，如果没有2、3两步，那么普通用户看不到任何信息。** ## 1、接入物理集群 @@ -36,4 +46,4 @@ ![op_add_logical_cluster](assets/op_add_logical_cluster.jpg) -如上图所示，填写逻辑集群信息，然后点击确定，即可完成逻辑集群的创建。 \ No newline at end of file +如上图所示，填写逻辑集群信息，然后点击确定，即可完成逻辑集群的创建。 diff --git a/docs/user_guide/add_cluster/assets/op_cluster_arch.png b/docs/user_guide/add_cluster/assets/op_cluster_arch.png new file mode 100644 index 00000000..aa972d9e Binary files /dev/null and b/docs/user_guide/add_cluster/assets/op_cluster_arch.png differ diff --git a/docs/user_guide/add_cluster/assets/op_cluster_flow.png b/docs/user_guide/add_cluster/assets/op_cluster_flow.png new file mode 100644 index 00000000..283f2676 Binary files /dev/null and b/docs/user_guide/add_cluster/assets/op_cluster_flow.png differ diff --git a/docs/user_guide/assets/resource_apply/production_consumption_flow.png b/docs/user_guide/assets/resource_apply/production_consumption_flow.png new file mode 100644 index 00000000..36187c83 Binary files /dev/null and b/docs/user_guide/assets/resource_apply/production_consumption_flow.png differ diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 28138a3a..8ab9781a 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -10,7 +10,7 @@ # FAQ - 0、Github图裂问题解决 -- 1、Topic申请时没有可选择的集群？ +- 1、Topic申请、新建监控告警等操作时没有可选择的集群？ - 2、逻辑集群 & Region的用途？ - 3、登录失败？ - 4、页面流量信息等无数据？ @@ -37,9 +37,13 @@ --- -### 1、Topic申请时没有可选择的集群？ +### 1、Topic申请、新建监控告警等操作时没有可选择的集群？ -- 参看 [kafka-manager 接入集群](docs/user_guide/add_cluster/add_cluster.md) 手册，这里的Region和逻辑集群都必须添加。 +缺少逻辑集群导致的，在Topic管理、监控告警、集群管理这三个Tab下面都是普通用户视角，普通用户看到的集群都是逻辑集群，因此在这三个Tab下进行操作时，都需要有逻辑集群。 + +逻辑集群的创建参看： + +- [kafka-manager 接入集群](docs/user_guide/add_cluster/add_cluster.md) 手册，这里的Region和逻辑集群都必须添加。 --- diff --git a/docs/user_guide/resource_apply/resource_apply.md b/docs/user_guide/resource_apply.md similarity index 92% rename from docs/user_guide/resource_apply/resource_apply.md rename to docs/user_guide/resource_apply.md index 427d8b14..87537f95 100644 --- a/docs/user_guide/resource_apply/resource_apply.md +++ b/docs/user_guide/resource_apply.md @@ -14,8 +14,8 @@ - 应用（App）：作为Kafka中的账户，使用AppID+password作为身份标识 - 集群：可使用平台提供的共享集群，也可为某一应用申请单独的集群 -- Topic：可申请创建Topic或申请其他Topic的生产/消费权限。进行生产/消费时通过Topic+AppID进行身份鉴权 -![production_consumption_flow](/production_consumption_flow.png) +- Topic：可申请创建Topic或申请其他Topic的生产/消费权限。进行生产/消费时通过Topic+AppID进行身份鉴权 +![production_consumption_flow](assets/resource_apply/production_consumption_flow.png) ## 应用申请 应用（App）作为Kafka中的账户，使用AppID+password作为身份标识。对Topic进行生产/消费时通过Topic+AppID进行身份鉴权。 diff --git a/kafka-manager-common/pom.xml b/kafka-manager-common/pom.xml index 67fbcdbd..6a8ff0cb 100644 --- a/kafka-manager-common/pom.xml +++ b/kafka-manager-common/pom.xml @@ -104,5 +104,10 @@ javax.servlet javax.servlet-api + + + junit + junit + \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/AccountRoleEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/AccountRoleEnum.java index 9c3cc06c..55412490 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/AccountRoleEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/AccountRoleEnum.java @@ -47,4 +47,13 @@ public enum AccountRoleEnum { } return AccountRoleEnum.UNKNOWN; } + + public static AccountRoleEnum getUserRoleEnum(String roleName) { + for (AccountRoleEnum elem: AccountRoleEnum.values()) { + if (elem.message.equalsIgnoreCase(roleName)) { + return elem; + } + } + return AccountRoleEnum.UNKNOWN; + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java new file mode 100644 index 00000000..bac44235 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java @@ -0,0 +1,32 @@ +package com.xiaojukeji.kafka.manager.common.bizenum; + +/** + * 过期Topic状态 + * @author zengqiao + * @date 21/01/25 + */ +public enum TopicExpiredStatusEnum { + ALREADY_NOTIFIED_AND_DELETED(-2, "已通知, 已下线"), + ALREADY_NOTIFIED_AND_CAN_DELETE(-1, "已通知, 可下线"), + ALREADY_EXPIRED_AND_WAIT_NOTIFY(0, "已过期, 待通知"), + ALREADY_NOTIFIED_AND_WAIT_RESPONSE(1, "已通知, 待反馈"), + + ; + + private int status; + + private String message; + + TopicExpiredStatusEnum(int status, String message) { + this.status = status; + this.message = message; + } + + public int getStatus() { + return status; + } + + public String getMessage() { + return message; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java index 0fb38302..323e9ec9 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java @@ -97,7 +97,7 @@ public class Result implements Serializable { return result; } - public static Result buildFailure(String message) { + public static Result buildGatewayFailure(String message) { Result result = new Result(); result.setCode(ResultStatus.GATEWAY_INVALID_REQUEST.getCode()); result.setMessage(message); @@ -105,6 +105,14 @@ public class Result implements Serializable { return result; } + public static Result buildFailure(String message) { + Result result = new Result(); + result.setCode(ResultStatus.FAIL.getCode()); + result.setMessage(message); + result.setData(null); + return result; + } + public static Result buildFrom(ResultStatus resultStatus) { Result result = new Result(); result.setCode(resultStatus.getCode()); diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java index 76e3aca8..94acb56d 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ResultStatus.java @@ -12,6 +12,8 @@ public enum ResultStatus { SUCCESS(Constant.SUCCESS, "success"), + FAIL(1, "操作失败"), + /** * 操作错误[1000, 2000) * ------------------------------------------------------------------------------------------ @@ -91,6 +93,8 @@ public enum ResultStatus { ZOOKEEPER_CONNECT_FAILED(8020, "zookeeper connect failed"), ZOOKEEPER_READ_FAILED(8021, "zookeeper read failed"), + ZOOKEEPER_WRITE_FAILED(8022, "zookeeper write failed"), + ZOOKEEPER_DELETE_FAILED(8023, "zookeeper delete failed"), // 调用集群任务里面的agent失败 CALL_CLUSTER_TASK_AGENT_FAILED(8030, " call cluster task agent failed"), diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/ClusterDetailDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/ClusterDetailDTO.java index 937d9cf8..2e903485 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/ClusterDetailDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/ClusterDetailDTO.java @@ -23,6 +23,8 @@ public class ClusterDetailDTO { private String securityProperties; + private String jmxProperties; + private Integer status; private Date gmtCreate; @@ -103,6 +105,14 @@ public class ClusterDetailDTO { this.securityProperties = securityProperties; } + public String getJmxProperties() { + return jmxProperties; + } + + public void setJmxProperties(String jmxProperties) { + this.jmxProperties = jmxProperties; + } + public Integer getStatus() { return status; } @@ -176,8 +186,9 @@ public class ClusterDetailDTO { ", bootstrapServers='" + bootstrapServers + '\'' + ", kafkaVersion='" + kafkaVersion + '\'' + ", idc='" + idc + '\'' + - ", mode='" + mode + '\'' + + ", mode=" + mode + ", securityProperties='" + securityProperties + '\'' + + ", jmxProperties='" + jmxProperties + '\'' + ", status=" + status + ", gmtCreate=" + gmtCreate + ", gmtModify=" + gmtModify + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/config/SinkTopicRequestTimeMetricsConfig.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/config/SinkTopicRequestTimeMetricsConfig.java deleted file mode 100644 index 91faaba1..00000000 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/config/SinkTopicRequestTimeMetricsConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.xiaojukeji.kafka.manager.common.entity.ao.config; - -/** - * @author zengqiao - * @date 20/9/7 - */ -public class SinkTopicRequestTimeMetricsConfig { - private Long clusterId; - - private String topicName; - - private Long startId; - - private Long step; - - public Long getClusterId() { - return clusterId; - } - - public void setClusterId(Long clusterId) { - this.clusterId = clusterId; - } - - public String getTopicName() { - return topicName; - } - - public void setTopicName(String topicName) { - this.topicName = topicName; - } - - public Long getStartId() { - return startId; - } - - public void setStartId(Long startId) { - this.startId = startId; - } - - public Long getStep() { - return step; - } - - public void setStep(Long step) { - this.step = step; - } - - @Override - public String toString() { - return "SinkTopicRequestTimeMetricsConfig{" + - "clusterId=" + clusterId + - ", topicName='" + topicName + '\'' + - ", startId=" + startId + - ", step=" + step + - '}'; - } -} \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/ControllerPreferredCandidateDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/ControllerPreferredCandidateDTO.java new file mode 100644 index 00000000..1b4c95b9 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/ControllerPreferredCandidateDTO.java @@ -0,0 +1,45 @@ +package com.xiaojukeji.kafka.manager.common.entity.dto.op; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +/** + * @author zengqiao + * @date 21/01/24 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@ApiModel(description="优选为Controller的候选者") +public class ControllerPreferredCandidateDTO { + @ApiModelProperty(value="集群ID") + private Long clusterId; + + @ApiModelProperty(value="优选为controller的BrokerId") + private List brokerIdList; + + public Long getClusterId() { + return clusterId; + } + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } + + public List getBrokerIdList() { + return brokerIdList; + } + + public void setBrokerIdList(List brokerIdList) { + this.brokerIdList = brokerIdList; + } + + @Override + public String toString() { + return "ControllerPreferredCandidateDTO{" + + "clusterId=" + clusterId + + ", brokerIdList=" + brokerIdList + + '}'; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/rd/ClusterDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/rd/ClusterDTO.java index 0b6fcebb..7afc09c6 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/rd/ClusterDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/rd/ClusterDTO.java @@ -102,12 +102,11 @@ public class ClusterDTO { '}'; } - public Boolean legal() { + public boolean legal() { if (ValidateUtils.isNull(clusterName) || ValidateUtils.isNull(zookeeper) || ValidateUtils.isNull(idc) - || ValidateUtils.isNull(bootstrapServers) - ) { + || ValidateUtils.isNull(bootstrapServers)) { return false; } return true; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/gateway/GatewayConfigDO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/gateway/GatewayConfigDO.java index c0e96000..fa29c7cf 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/gateway/GatewayConfigDO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/gateway/GatewayConfigDO.java @@ -17,6 +17,8 @@ public class GatewayConfigDO { private Long version; + private String description; + private Date createTime; private Date modifyTime; @@ -61,6 +63,14 @@ public class GatewayConfigDO { this.version = version; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + public Date getCreateTime() { return createTime; } @@ -85,6 +95,7 @@ public class GatewayConfigDO { ", name='" + name + '\'' + ", value='" + value + '\'' + ", version=" + version + + ", description='" + description + '\'' + ", createTime=" + createTime + ", modifyTime=" + modifyTime + '}'; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/op/expert/ExpiredTopicVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/op/expert/ExpiredTopicVO.java index 46c7a3a2..c4921259 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/op/expert/ExpiredTopicVO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/op/expert/ExpiredTopicVO.java @@ -28,7 +28,7 @@ public class ExpiredTopicVO { @ApiModelProperty(value = "负责人") private String principals; - @ApiModelProperty(value = "状态, -1:可下线, 0:过期待通知, 1+:已通知待反馈") + @ApiModelProperty(value = "状态, -1:已通知可下线, 0:过期待通知, 1+:已通知待反馈") private Integer status; public Long getClusterId() { diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java index a0b402af..72314c31 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java @@ -26,6 +26,9 @@ public class GatewayConfigVO { @ApiModelProperty(value="版本") private Long version; + @ApiModelProperty(value="描述说明") + private String description; + @ApiModelProperty(value="创建时间") private Date createTime; @@ -72,6 +75,14 @@ public class GatewayConfigVO { this.version = version; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + public Date getCreateTime() { return createTime; } @@ -96,6 +107,7 @@ public class GatewayConfigVO { ", name='" + name + '\'' + ", value='" + value + '\'' + ", version=" + version + + ", description='" + description + '\'' + ", createTime=" + createTime + ", modifyTime=" + modifyTime + '}'; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java index 46d177ad..283d59c5 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java @@ -60,6 +60,13 @@ public class JsonUtils { return JSON.parseObject(src, clazz); } + public static List stringToArrObj(String src, Class clazz) { + if (ValidateUtils.isBlank(src)) { + return null; + } + return JSON.parseArray(src, clazz); + } + public static List parseTopicConnections(Long clusterId, JSONObject jsonObject, long postTime) { List connectionDOList = new ArrayList<>(); for (String clientType: jsonObject.keySet()) { diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java index 1ece8f9f..6bd0c55c 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java @@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.common.utils; import org.apache.commons.lang.StringUtils; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -11,6 +12,20 @@ import java.util.Set; * @date 20/4/16 */ public class ValidateUtils { + /** + * 任意一个为空, 则返回true + */ + public static boolean anyNull(Object... objects) { + return Arrays.stream(objects).anyMatch(ValidateUtils::isNull); + } + + /** + * 是空字符串或者空 + */ + public static boolean anyBlank(String... strings) { + return Arrays.stream(strings).anyMatch(StringUtils::isBlank); + } + /** * 为空 */ diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java index fc70c6b2..c7c69ca3 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/jmx/JmxConnectorWrap.java @@ -79,7 +79,7 @@ public class JmxConnectorWrap { try { Map environment = new HashMap(); if (!ValidateUtils.isBlank(this.jmxConfig.getUsername()) && !ValidateUtils.isBlank(this.jmxConfig.getPassword())) { - environment.put(javax.management.remote.JMXConnector.CREDENTIALS, Arrays.asList(this.jmxConfig.getUsername(), this.jmxConfig.getPassword())); + environment.put(JMXConnector.CREDENTIALS, Arrays.asList(this.jmxConfig.getUsername(), this.jmxConfig.getPassword())); } if (jmxConfig.isOpenSSL() != null && this.jmxConfig.isOpenSSL()) { environment.put(Context.SECURITY_PROTOCOL, "ssl"); diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java new file mode 100644 index 00000000..2419901a --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ldap/LDAPAuthentication.java @@ -0,0 +1,120 @@ +package com.xiaojukeji.kafka.manager.common.utils.ldap; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.naming.AuthenticationException; +import javax.naming.Context; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.SearchControls; +import javax.naming.directory.SearchResult; +import javax.naming.ldap.InitialLdapContext; +import javax.naming.ldap.LdapContext; +import java.util.Hashtable; + +@Component +public class LDAPAuthentication { + + @Value(value = "${ldap.url}") + private String ldapUrl; + + @Value(value = "${ldap.basedn}") + private String ldapBasedn; + + @Value(value = "${ldap.factory}") + private String ldapFactory; + + @Value(value = "${ldap.auth-user-registration-role}") + private String authUserRegistrationRole; + + @Value(value = "${ldap.security.authentication}") + private String securityAuthentication; + + @Value(value = "${ldap.security.principal}") + private String securityPrincipal; + + @Value(value = "${ldap.security.credentials}") + private String securityCredentials; + + private LdapContext getConnect() { + Hashtable env = new Hashtable(); + env.put(Context.INITIAL_CONTEXT_FACTORY, ldapFactory); + env.put(Context.PROVIDER_URL, ldapUrl + ldapBasedn); + env.put(Context.SECURITY_AUTHENTICATION, securityAuthentication); + + // 此处若不指定用户名和密码,则自动转换为匿名登录 + env.put(Context.SECURITY_PRINCIPAL, securityPrincipal); + env.put(Context.SECURITY_CREDENTIALS, securityCredentials); + try { + return new InitialLdapContext(env, null); + } catch (AuthenticationException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + private String getUserDN(String account,LdapContext ctx) { + String userDN = ""; + try { + SearchControls constraints = new SearchControls(); + constraints.setSearchScope(SearchControls.SUBTREE_SCOPE); + NamingEnumeration en = ctx.search("", "account=" + account, constraints); + if (en == null || !en.hasMoreElements()) { + return ""; + } + // maybe more than one element + while (en.hasMoreElements()) { + Object obj = en.nextElement(); + if (obj instanceof SearchResult) { + SearchResult si = (SearchResult) obj; + userDN += si.getName(); + userDN += "," + ldapBasedn; + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + return userDN; + } + + /** + * LDAP账密验证 + * @param account + * @param password + * @return + */ + public boolean authenricate(String account, String password) { + LdapContext ctx = getConnect(); + + boolean valide = false; + + try { + String userDN = getUserDN(account,ctx); + + ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN); + ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password); + ctx.reconnect(null); + valide = true; + } catch (AuthenticationException e) { + System.out.println(e.toString()); + } catch (NamingException e) { + e.printStackTrace(); + }finally { + if(ctx!=null) { + try { + ctx.close(); + } catch (NamingException e) { + e.printStackTrace(); + } + } + } + + return valide; + } + +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java index e0a5632a..6705f435 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java @@ -33,7 +33,9 @@ public class ZkPathUtil { private static final String D_METRICS_CONFIG_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "KafkaExMetrics"; - public static final String D_CONTROLLER_CANDIDATES = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "extension/candidates"; + public static final String D_CONFIG_EXTENSION_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "extension"; + + public static final String D_CONTROLLER_CANDIDATES = D_CONFIG_EXTENSION_ROOT_NODE + ZOOKEEPER_SEPARATOR + "candidates"; public static String getBrokerIdNodePath(Integer brokerId) { return BROKER_IDS_ROOT + ZOOKEEPER_SEPARATOR + String.valueOf(brokerId); @@ -111,6 +113,10 @@ public class ZkPathUtil { } public static String getKafkaExtraMetricsPath(Integer brokerId) { - return D_METRICS_CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + String.valueOf(brokerId); + return D_METRICS_CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + brokerId; + } + + public static String getControllerCandidatePath(Integer brokerId) { + return D_CONTROLLER_CANDIDATES + ZOOKEEPER_SEPARATOR + brokerId; } } diff --git a/kafka-manager-common/src/test/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtilsTest.java b/kafka-manager-common/src/test/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtilsTest.java new file mode 100644 index 00000000..1d338015 --- /dev/null +++ b/kafka-manager-common/src/test/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtilsTest.java @@ -0,0 +1,18 @@ +package com.xiaojukeji.kafka.manager.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class JsonUtilsTest { + @Test + public void testMapToJsonString() { + Map map = new HashMap<>(); + map.put("key", "value"); + map.put("int", 1); + String expectRes = "{\"key\":\"value\",\"int\":1}"; + Assert.assertEquals(expectRes, JsonUtils.toJSONString(map)); + } +} diff --git a/kafka-manager-console/src/component/antd/index.tsx b/kafka-manager-console/src/component/antd/index.tsx index 2d771efe..d0958daf 100644 --- a/kafka-manager-console/src/component/antd/index.tsx +++ b/kafka-manager-console/src/component/antd/index.tsx @@ -94,6 +94,9 @@ import 'antd/es/divider/style'; import Upload from 'antd/es/upload'; import 'antd/es/upload/style'; +import Transfer from 'antd/es/transfer'; +import 'antd/es/transfer/style'; + import TimePicker from 'antd/es/time-picker'; import 'antd/es/time-picker/style'; @@ -142,5 +145,6 @@ export { TimePicker, RangePickerValue, Badge, - Popover + Popover, + Transfer }; diff --git a/kafka-manager-console/src/component/editor/index.less b/kafka-manager-console/src/component/editor/index.less index 4ff05854..36c52cde 100644 --- a/kafka-manager-console/src/component/editor/index.less +++ b/kafka-manager-console/src/component/editor/index.less @@ -25,7 +25,7 @@ .editor{ height: 100%; position: absolute; - left: -14%; + left: -12%; width: 120%; } } diff --git a/kafka-manager-console/src/component/editor/monacoEditor.tsx b/kafka-manager-console/src/component/editor/monacoEditor.tsx index 7a0dd44c..ac0a297a 100644 --- a/kafka-manager-console/src/component/editor/monacoEditor.tsx +++ b/kafka-manager-console/src/component/editor/monacoEditor.tsx @@ -21,24 +21,12 @@ class Monacoeditor extends React.Component { public state = { placeholder: '', }; - // public arr = '{"clusterId":95,"startId":37397856,"step":100,"topicName":"kmo_topic_metrics_tempory_zq"}'; - // public Ars(a: string) { - // const obj = JSON.parse(a); - // const newobj: any = {}; - // for (const item in obj) { - // if (typeof obj[item] === 'object') { - // this.Ars(obj[item]); - // } else { - // newobj[item] = obj[item]; - // } - // } - // return JSON.stringify(newobj); - // } + public async componentDidMount() { const { value, onChange } = this.props; const format: any = await format2json(value); this.editor = monaco.editor.create(this.ref, { - value: format.result, + value: format.result || value, language: 'json', lineNumbers: 'off', scrollBeyondLastLine: false, @@ -48,7 +36,7 @@ class Monacoeditor extends React.Component { minimap: { enabled: false, }, - // automaticLayout: true, // 自动布局 + automaticLayout: true, // 自动布局 glyphMargin: true, // 字形边缘 {},[] // useTabStops: false, // formatOnPaste: true, diff --git a/kafka-manager-console/src/component/x-form/index.tsx b/kafka-manager-console/src/component/x-form/index.tsx index 9530f2bb..dc435d0f 100755 --- a/kafka-manager-console/src/component/x-form/index.tsx +++ b/kafka-manager-console/src/component/x-form/index.tsx @@ -130,6 +130,8 @@ class XForm extends React.Component { this.renderFormItem(formItem), )} {formItem.renderExtraElement ? formItem.renderExtraElement() : null} + {/* 添加保存时间提示文案 */} + {formItem.attrs?.prompttype ? {formItem.attrs.prompttype} : null} ); })} diff --git a/kafka-manager-console/src/constants/strategy.ts b/kafka-manager-console/src/constants/strategy.ts index c0d19001..e92563e6 100644 --- a/kafka-manager-console/src/constants/strategy.ts +++ b/kafka-manager-console/src/constants/strategy.ts @@ -67,7 +67,7 @@ export const timeMonthStr = 'YYYY/MM'; // tslint:disable-next-line:max-line-length export const indexUrl ={ - indexUrl:'https://github.com/didi/kafka-manager', + indexUrl:'https://github.com/didi/Logi-KafkaManager/blob/master/docs/user_guide/kafka_metrics_desc.md', // 指标说明 cagUrl:'https://github.com/didi/Logi-KafkaManager/blob/master/docs/user_guide/add_cluster/add_cluster.md', // 集群接入指南 Cluster access Guide } diff --git a/kafka-manager-console/src/container/admin/cluster-detail/cluster-consumer.tsx b/kafka-manager-console/src/container/admin/cluster-detail/cluster-consumer.tsx index 5605a372..911f44d2 100644 --- a/kafka-manager-console/src/container/admin/cluster-detail/cluster-consumer.tsx +++ b/kafka-manager-console/src/container/admin/cluster-detail/cluster-consumer.tsx @@ -100,7 +100,7 @@ export class ClusterConsumer extends SearchAndFilterContainer {
  • {this.props.tab}
    • - {this.renderSearch()} + {this.renderSearch('', '请输入消费组名称')}
(origin: T[]) { + let data: T[] = origin; + let { searchCandidateKey } = this.state; + searchCandidateKey = (searchCandidateKey + '').trim().toLowerCase(); + + data = searchCandidateKey ? origin.filter((item: IController) => + (item.host !== undefined && item.host !== null) && item.host.toLowerCase().includes(searchCandidateKey as string), + ) : origin; + return data; + } + + // 候选controller + public renderCandidateController() { + const columns = [ + { + title: 'BrokerId', + dataIndex: 'brokerId', + key: 'brokerId', + width: '20%', + sorter: (a: IController, b: IController) => b.brokerId - a.brokerId, + render: (r: string, t: IController) => { + return ( + {r} + + ); + }, + }, + { + title: 'BrokerHost', + key: 'host', + dataIndex: 'host', + width: '20%', + // render: (r: string, t: IController) => { + // return ( + // {r} + // + // ); + // }, + }, + { + title: 'Broker状态', + key: 'status', + dataIndex: 'status', + width: '20%', + render: (r: number, t: IController) => { + return ( + {r === 1 ? '不在线' : '在线'} + ); + }, + }, + { + title: '创建时间', + dataIndex: 'startTime', + key: 'startTime', + width: '25%', + sorter: (a: IController, b: IController) => b.timestamp - a.timestamp, + render: (t: number) => moment(t).format(timeFormat), + }, + { + title: '操作', + dataIndex: 'operation', + key: 'operation', + width: '15%', + render: (r: string, t: IController) => { + return ( + this.deleteCandidateCancel(t)} + cancelText="取消" + okText="确认" + > + 删除 + + ); + }, + }, + ]; + + return ( +
+ ); + } + public renderController() { const columns = [ @@ -58,12 +151,6 @@ export class ClusterController extends SearchAndFilterContainer { key: 'host', dataIndex: 'host', width: '30%', - // render: (r: string, t: IController) => { - // return ( - // {r} - // - // ); - // }, }, { title: '变更时间', @@ -87,16 +174,104 @@ export class ClusterController extends SearchAndFilterContainer { public componentDidMount() { admin.getControllerHistory(this.clusterId); + admin.getCandidateController(this.clusterId); + admin.getBrokersMetadata(this.clusterId); + } + + public addController = () => { + this.setState({ isCandidateModel: true, targetKeys: [] }) + } + + public addCandidateChange = (targetKeys: any) => { + this.setState({ targetKeys }) + } + + + + public handleCandidateCancel = () => { + this.setState({ isCandidateModel: false }); + } + + public handleCandidateOk = () => { + let brokerIdList = this.state.targetKeys.map((item: any) => { + return admin.brokersMetadata[item].brokerId + }) + admin.addCandidateController(this.clusterId, brokerIdList).then(data => { + notification.success({ message: '新增成功' }); + admin.getCandidateController(this.clusterId); + }).catch(err => { + notification.error({ message: '新增失败' }); + }) + this.setState({ isCandidateModel: false, targetKeys: [] }); + } + + public deleteCandidateCancel = (target: any) => { + admin.deleteCandidateCancel(this.clusterId, [target.brokerId]).then(() => { + notification.success({ message: '删除成功' }); + }); + this.setState({ isCandidateModel: false }); + } + + public renderAddCandidateController() { + let filterControllerCandidate = admin.brokersMetadata.filter((item: any) => { + return !admin.filtercontrollerCandidate.includes(item.brokerId) + }) + + return ( + this.handleCandidateOk()} + onCancel={() => this.handleCandidateCancel()} + footer={<> + + + + } + > + item.host} + onChange={(targetKeys) => this.addCandidateChange(targetKeys)} + titles={['未选', '已选']} + locale={{ + itemUnit: '项', + itemsUnit: '项', + }} + listStyle={{ + width: "45%", + }} + /> + + ); } public render() { return (
    +
  • + 候选Controller + Controller将会优先从以下Broker中选举 +
    • +
    +
    + +
    + {this.renderSearch('', '请查找Host', 'searchCandidateKey')} +
    +
+ {this.renderCandidateController()} +
  • {this.props.tab}
    • {this.renderSearch('', '请输入Host')}
{this.renderController()} + {this.renderAddCandidateController()}
); } diff --git a/kafka-manager-console/src/container/admin/cluster-detail/index.less b/kafka-manager-console/src/container/admin/cluster-detail/index.less index 65c45b9c..0dd4d106 100644 --- a/kafka-manager-console/src/container/admin/cluster-detail/index.less +++ b/kafka-manager-console/src/container/admin/cluster-detail/index.less @@ -94,4 +94,10 @@ .region-prompt{ font-weight: bold; text-align: center; +} + +.asd{ + display: flex; + justify-content: space-around; + align-items: center; } \ No newline at end of file diff --git a/kafka-manager-console/src/container/admin/cluster-detail/index.tsx b/kafka-manager-console/src/container/admin/cluster-detail/index.tsx index 5882dd57..027dde27 100644 --- a/kafka-manager-console/src/container/admin/cluster-detail/index.tsx +++ b/kafka-manager-console/src/container/admin/cluster-detail/index.tsx @@ -32,9 +32,9 @@ export class ClusterDetail extends React.Component { } public render() { - let content = {} as IMetaData; - content = admin.basicInfo ? admin.basicInfo : content; - return ( + let content = {} as IMetaData; + content = admin.basicInfo ? admin.basicInfo : content; + return ( <> - + @@ -60,11 +60,11 @@ export class ClusterDetail extends React.Component { - + - - + + diff --git a/kafka-manager-console/src/container/admin/cluster-list/index.tsx b/kafka-manager-console/src/container/admin/cluster-list/index.tsx index 63ccd93e..dfac45d7 100644 --- a/kafka-manager-console/src/container/admin/cluster-list/index.tsx +++ b/kafka-manager-console/src/container/admin/cluster-list/index.tsx @@ -12,6 +12,7 @@ import { urlPrefix } from 'constants/left-menu'; import { indexUrl } from 'constants/strategy' import { region } from 'store'; import './index.less'; +import Monacoeditor from 'component/editor/monacoEditor'; import { getAdminClusterColumns } from '../config'; const { confirm } = Modal; @@ -132,6 +133,25 @@ export class ClusterList extends SearchAndFilterContainer { "security.protocol": "SASL_PLAINTEXT", "sasl.mechanism": "PLAIN", "sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\\"xxxxxx\\" password=\\"xxxxxx\\";" +}`, + rows: 8, + }, + }, + { + key: 'jmxProperties', + label: 'JMX认证', + type: 'text_area', + rules: [{ + required: false, + message: '请输入JMX认证', + }], + attrs: { + placeholder: `请输入JMX认证，例如： +{ +"maxConn": 10, #KM对单台Broker对最大连接数 +"username": "xxxxx", #用户名 +"password": "xxxxx", #密码 +"openSSL": true, #开启SSL，true表示开启SSL，false表示关闭 }`, rows: 8, }, diff --git a/kafka-manager-console/src/container/admin/config.tsx b/kafka-manager-console/src/container/admin/config.tsx index bce02cce..09a70f83 100644 --- a/kafka-manager-console/src/container/admin/config.tsx +++ b/kafka-manager-console/src/container/admin/config.tsx @@ -1,8 +1,8 @@ import * as React from 'react'; -import { IUser, IUploadFile, IConfigure, IMetaData, IBrokersPartitions } from 'types/base-type'; +import { IUser, IUploadFile, IConfigure, IConfigGateway, IMetaData } from 'types/base-type'; import { users } from 'store/users'; import { version } from 'store/version'; -import { showApplyModal, showModifyModal, showConfigureModal } from 'container/modal/admin'; +import { showApplyModal, showApplyModalModifyPassword, showModifyModal, showConfigureModal, showConfigGatewayModal } from 'container/modal/admin'; import { Popconfirm, Tooltip } from 'component/antd'; import { admin } from 'store/admin'; import { cellStyle } from 'constants/table'; @@ -27,6 +27,7 @@ export const getUserColumns = () => { return ( showApplyModal(record)}>编辑 + showApplyModalModifyPassword(record)}>修改密码 users.deleteUser(record.username)} @@ -184,6 +185,87 @@ export const getConfigureColumns = () => { return columns; }; +// 网关配置 +export const getConfigColumns = () => { + const columns = [ + { + title: '配置类型', + dataIndex: 'type', + key: 'type', + width: '25%', + ellipsis: true, + sorter: (a: IConfigGateway, b: IConfigGateway) => a.type.charCodeAt(0) - b.type.charCodeAt(0), + }, + { + title: '配置键', + dataIndex: 'name', + key: 'name', + width: '15%', + ellipsis: true, + sorter: (a: IConfigGateway, b: IConfigGateway) => a.name.charCodeAt(0) - b.name.charCodeAt(0), + }, + { + title: '配置值', + dataIndex: 'value', + key: 'value', + width: '20%', + ellipsis: true, + sorter: (a: IConfigGateway, b: IConfigGateway) => a.value.charCodeAt(0) - b.value.charCodeAt(0), + render: (t: string) => { + return t.substr(0, 1) === '{' && t.substr(0, -1) === '}' ? JSON.stringify(JSON.parse(t), null, 4) : t; + }, + }, + { + title: '修改时间', + dataIndex: 'modifyTime', + key: 'modifyTime', + width: '15%', + sorter: (a: IConfigGateway, b: IConfigGateway) => b.modifyTime - a.modifyTime, + render: (t: number) => moment(t).format(timeFormat), + }, + { + title: '版本号', + dataIndex: 'version', + key: 'version', + width: '10%', + ellipsis: true, + sorter: (a: IConfigGateway, b: IConfigGateway) => b.version.charCodeAt(0) - a.version.charCodeAt(0), + }, + { + title: '描述信息', + dataIndex: 'description', + key: 'description', + width: '20%', + ellipsis: true, + onCell: () => ({ + style: { + maxWidth: 180, + ...cellStyle, + }, + }), + }, + { + title: '操作', + width: '10%', + render: (text: string, record: IConfigGateway) => { + return ( + + showConfigGatewayModal(record)}>编辑 + admin.deleteConfigGateway({ id: record.id })} + cancelText="取消" + okText="确认" + > + 删除 + + ); + }, + }, + ]; + return columns; +}; + const renderClusterHref = (value: number | string, item: IMetaData, key: number) => { return ( // 0 暂停监控--不可点击 1 监控中---可正常点击 <> diff --git a/kafka-manager-console/src/container/admin/configure-management.tsx b/kafka-manager-console/src/container/admin/configure-management.tsx index 680d1da7..5c3494b9 100644 --- a/kafka-manager-console/src/container/admin/configure-management.tsx +++ b/kafka-manager-console/src/container/admin/configure-management.tsx @@ -3,11 +3,11 @@ import { SearchAndFilterContainer } from 'container/search-filter'; import { Table, Button, Spin } from 'component/antd'; import { admin } from 'store/admin'; import { observer } from 'mobx-react'; -import { IConfigure } from 'types/base-type'; +import { IConfigure, IConfigGateway } from 'types/base-type'; import { users } from 'store/users'; import { pagination } from 'constants/table'; -import { getConfigureColumns } from './config'; -import { showConfigureModal } from 'container/modal/admin'; +import { getConfigureColumns, getConfigColumns } from './config'; +import { showConfigureModal, showConfigGatewayModal } from 'container/modal/admin'; @observer export class ConfigureManagement extends SearchAndFilterContainer { @@ -17,7 +17,12 @@ export class ConfigureManagement extends SearchAndFilterContainer { }; public componentDidMount() { - admin.getConfigure(); + if (this.props.isShow) { + admin.getGatewayList(); + admin.getGatewayType(); + } else { + admin.getConfigure(); + } } public getData(origin: T[]) { @@ -34,15 +39,34 @@ export class ConfigureManagement extends SearchAndFilterContainer { return data; } + public getGatewayData(origin: T[]) { + let data: T[] = origin; + let { searchKey } = this.state; + searchKey = (searchKey + '').trim().toLowerCase(); + + data = searchKey ? origin.filter((item: IConfigGateway) => + ((item.name !== undefined && item.name !== null) && item.name.toLowerCase().includes(searchKey as string)) + || ((item.value !== undefined && item.value !== null) && item.value.toLowerCase().includes(searchKey as string)) + || ((item.description !== undefined && item.description !== null) && + item.description.toLowerCase().includes(searchKey as string)), + ) : origin; + return data; + } + public renderTable() { return ( -
:
+ />} ); @@ -53,7 +77,7 @@ export class ConfigureManagement extends SearchAndFilterContainer {
    {this.renderSearch('', '请输入配置键、值或描述')}
  • - +
); diff --git a/kafka-manager-console/src/container/admin/data-curve/index.tsx b/kafka-manager-console/src/container/admin/data-curve/index.tsx index bd113aeb..b822957c 100644 --- a/kafka-manager-console/src/container/admin/data-curve/index.tsx +++ b/kafka-manager-console/src/container/admin/data-curve/index.tsx @@ -6,6 +6,7 @@ import { curveKeys, CURVE_KEY_MAP, PERIOD_RADIO_MAP, PERIOD_RADIO } from './conf import moment = require('moment'); import { observer } from 'mobx-react'; import { timeStampStr } from 'constants/strategy'; +import { adminMonitor } from 'store/admin-monitor'; @observer export class DataCurveFilter extends React.Component { @@ -21,6 +22,7 @@ export class DataCurveFilter extends React.Component { } public refreshAll = () => { + adminMonitor.setRequestId(null); Object.keys(curveKeys).forEach((c: curveKeys) => { const { typeInfo, curveInfo: option } = CURVE_KEY_MAP.get(c); const { parser } = typeInfo; @@ -32,7 +34,7 @@ export class DataCurveFilter extends React.Component { return ( <> - {PERIOD_RADIO.map(p => {p.label})} + {PERIOD_RADIO.map(p => {p.label})} - - - - - - - - - - - + + + + + + + + + + + + + + ); } diff --git a/kafka-manager-console/src/container/admin/user-management.tsx b/kafka-manager-console/src/container/admin/user-management.tsx index 757ceabb..1dc38e06 100644 --- a/kafka-manager-console/src/container/admin/user-management.tsx +++ b/kafka-manager-console/src/container/admin/user-management.tsx @@ -29,7 +29,7 @@ export class UserManagement extends SearchAndFilterContainer { searchKey = (searchKey + '').trim().toLowerCase(); data = searchKey ? origin.filter((item: IUser) => - (item.username !== undefined && item.username !== null) && item.username.toLowerCase().includes(searchKey as string)) : origin ; + (item.username !== undefined && item.username !== null) && item.username.toLowerCase().includes(searchKey as string)) : origin; return data; } diff --git a/kafka-manager-console/src/container/alarm/add-alarm/alarm-select.tsx b/kafka-manager-console/src/container/alarm/add-alarm/alarm-select.tsx index 6d19ec26..5cd1f4f0 100644 --- a/kafka-manager-console/src/container/alarm/add-alarm/alarm-select.tsx +++ b/kafka-manager-console/src/container/alarm/add-alarm/alarm-select.tsx @@ -1,7 +1,7 @@ import * as React from 'react'; import { alarm } from 'store/alarm'; import { IMonitorGroups } from 'types/base-type'; -import { getValueFromLocalStorage, setValueToLocalStorage } from 'lib/local-storage'; +import { getValueFromLocalStorage, setValueToLocalStorage, deleteValueFromLocalStorage } from 'lib/local-storage'; import { VirtualScrollSelect } from '../../../component/virtual-scroll-select'; interface IAlarmSelectProps { @@ -36,6 +36,10 @@ export class AlarmSelect extends React.Component { onChange && onChange(params); } + public componentWillUnmount() { + deleteValueFromLocalStorage('monitorGroups'); + } + public render() { const { value, isDisabled } = this.props; return ( diff --git a/kafka-manager-console/src/container/alarm/alarm-list.tsx b/kafka-manager-console/src/container/alarm/alarm-list.tsx index 6dd6680b..54d266f1 100644 --- a/kafka-manager-console/src/container/alarm/alarm-list.tsx +++ b/kafka-manager-console/src/container/alarm/alarm-list.tsx @@ -9,6 +9,7 @@ import { pagination } from 'constants/table'; import { urlPrefix } from 'constants/left-menu'; import { alarm } from 'store/alarm'; import 'styles/table-filter.less'; +import { Link } from 'react-router-dom'; @observer export class AlarmList extends SearchAndFilterContainer { @@ -24,7 +25,7 @@ export class AlarmList extends SearchAndFilterContainer { if (app.active !== '-1' || searchKey !== '') { data = origin.filter(d => ((d.name !== undefined && d.name !== null) && d.name.toLowerCase().includes(searchKey as string) - || ((d.operator !== undefined && d.operator !== null) && d.operator.toLowerCase().includes(searchKey as string))) + || ((d.operator !== undefined && d.operator !== null) && d.operator.toLowerCase().includes(searchKey as string))) && (app.active === '-1' || d.appId === (app.active + '')), ); } else { @@ -55,9 +56,7 @@ export class AlarmList extends SearchAndFilterContainer { {this.renderSearch('名称：', '请输入告警规则或者操作人')}
    • @@ -68,6 +67,9 @@ export class AlarmList extends SearchAndFilterContainer { if (!alarm.monitorStrategies.length) { alarm.getMonitorStrategies(); } + if (!app.data.length) { + app.getAppList(); + } } public render() { diff --git a/kafka-manager-console/src/container/cluster/my-cluster.tsx b/kafka-manager-console/src/container/cluster/my-cluster.tsx index 3c54763a..e017b0dd 100644 --- a/kafka-manager-console/src/container/cluster/my-cluster.tsx +++ b/kafka-manager-console/src/container/cluster/my-cluster.tsx @@ -91,7 +91,7 @@ export class MyCluster extends SearchAndFilterContainer { ], formData: {}, visible: true, - title: '申请集群', + title:
    申请集群资源申请文档
    , okText: '确认', onSubmit: (value: any) => { value.idc = region.currentRegion; diff --git a/kafka-manager-console/src/container/drawer/data-migration.tsx b/kafka-manager-console/src/container/drawer/data-migration.tsx index 28353004..4da64f5c 100644 --- a/kafka-manager-console/src/container/drawer/data-migration.tsx +++ b/kafka-manager-console/src/container/drawer/data-migration.tsx @@ -117,12 +117,12 @@ class DataMigrationFormTable extends React.Component { key: 'maxThrottle', editable: true, }, { - title: '迁移保存时间(h)', + title: '迁移后Topic保存时间(h)', dataIndex: 'reassignRetentionTime', key: 'reassignRetentionTime', editable: true, }, { - title: '原本保存时间(h)', + title: '原Topic保存时间(h)', dataIndex: 'retentionTime', key: 'retentionTime', // originalRetentionTime width: '132px', diff --git a/kafka-manager-console/src/container/modal/admin/cluster.ts b/kafka-manager-console/src/container/modal/admin/cluster.ts index cea987f6..20ed9098 100644 --- a/kafka-manager-console/src/container/modal/admin/cluster.ts +++ b/kafka-manager-console/src/container/modal/admin/cluster.ts @@ -61,6 +61,7 @@ export const showEditClusterTopic = (item: IClusterTopics) => { attrs: { placeholder: '请输入保存时间', suffix: '小时', + prompttype:'修改保存时间，预计一分钟左右生效!' }, }, { diff --git a/kafka-manager-console/src/container/modal/admin/task.ts b/kafka-manager-console/src/container/modal/admin/task.ts index be240f7a..d9a609ac 100644 --- a/kafka-manager-console/src/container/modal/admin/task.ts +++ b/kafka-manager-console/src/container/modal/admin/task.ts @@ -158,26 +158,26 @@ export const createMigrationTasks = () => { }, { key: 'originalRetentionTime', - label: '原本保存时间', + label: '原Topic保存时间', rules: [{ required: true, - message: '请输入原本保存时间', + message: '请输入原Topic保存时间', }], attrs: { disabled: true, - placeholder: '请输入原本保存时间', + placeholder: '请输入原Topic保存时间', suffix: '小时', }, }, { key: 'reassignRetentionTime', - label: '迁移保存时间', + label: '迁移后Topic保存时间', rules: [{ required: true, - message: '请输入迁移保存时间', + message: '请输入迁移后Topic保存时间', }], attrs: { - placeholder: '请输入迁移保存时间', + placeholder: '请输入迁移后Topic保存时间', suffix: '小时', }, }, diff --git a/kafka-manager-console/src/container/modal/admin/user.ts b/kafka-manager-console/src/container/modal/admin/user.ts index 9f35e4cf..51ca360d 100644 --- a/kafka-manager-console/src/container/modal/admin/user.ts +++ b/kafka-manager-console/src/container/modal/admin/user.ts @@ -24,26 +24,111 @@ export const showApplyModal = (record?: IUser) => { value: +item, })), rules: [{ required: true, message: '请选择角色' }], - }, { - key: 'password', - label: '密码', - type: FormItemType.inputPassword, - rules: [{ required: !record, message: '请输入密码' }], - }, + }, + // { + // key: 'password', + // label: '密码', + // type: FormItemType.inputPassword, + // rules: [{ required: !record, message: '请输入密码' }], + // }, ], formData: record || {}, visible: true, title: record ? '修改用户' : '新增用户', onSubmit: (value: IUser) => { if (record) { - return users.modfiyUser(value).then(() => { - message.success('操作成功'); - }); + return users.modfiyUser(value) } return users.addUser(value).then(() => { message.success('操作成功'); }); }, }; + if(!record){ + let formMap: any = xFormModal.formMap + formMap.splice(2, 0,{ + key: 'password', + label: '密码', + type: FormItemType.inputPassword, + rules: [{ required: !record, message: '请输入密码' }], + },) + } wrapper.open(xFormModal); }; + +// const handleCfPassword = (rule:any, value:any, callback:any)=>{ +// if() +// } +export const showApplyModalModifyPassword = (record: IUser) => { + const xFormModal:any = { + formMap: [ + // { + // key: 'oldPassword', + // label: '旧密码', + // type: FormItemType.inputPassword, + // rules: [{ + // required: true, + // message: '请输入旧密码', + // }] + // }, + { + key: 'newPassword', + label: '新密码', + type: FormItemType.inputPassword, + rules: [ + { + required: true, + message: '请输入新密码', + } + ], + attrs:{ + onChange:(e:any)=>{ + users.setNewPassWord(e.target.value) + } + } + }, + { + key: 'confirmPassword', + label: '确认密码', + type: FormItemType.inputPassword, + rules: [ + { + required: true, + message: '请确认密码', + validator:(rule:any, value:any, callback:any) => { + // 验证新密码的一致性 + if(users.newPassWord){ + if(value!==users.newPassWord){ + rule.message = "两次密码输入不一致"; + callback('两次密码输入不一致') + }else{ + callback() + } + }else if(!value){ + rule.message = "请确认密码"; + callback('请确认密码'); + }else{ + callback() + } + }, + } + ], + }, + ], + formData: record || {}, + visible: true, + title: '修改密码', + onSubmit: (value: IUser) => { + let params:any = { + username:record?.username, + password:value.confirmPassword, + role:record?.role, + } + return users.modfiyUser(params).then(() => { + message.success('操作成功'); + }); + }, + } + wrapper.open(xFormModal); +}; + diff --git a/kafka-manager-console/src/container/modal/admin/version.ts b/kafka-manager-console/src/container/modal/admin/version.ts index ea642a8f..c863eba1 100644 --- a/kafka-manager-console/src/container/modal/admin/version.ts +++ b/kafka-manager-console/src/container/modal/admin/version.ts @@ -1,6 +1,6 @@ import * as React from 'react'; -import { notification } from 'component/antd'; -import { IUploadFile, IConfigure } from 'types/base-type'; +import { notification, Select } from 'component/antd'; +import { IUploadFile, IConfigure, IConfigGateway } from 'types/base-type'; import { version } from 'store/version'; import { admin } from 'store/admin'; import { wrapper } from 'store'; @@ -97,8 +97,8 @@ const updateFormModal = (type: number) => { formMap[2].attrs = { accept: version.fileSuffix, }, - // tslint:disable-next-line:no-unused-expression - wrapper.ref && wrapper.ref.updateFormMap$(formMap, wrapper.xFormWrapper.formData, true); + // tslint:disable-next-line:no-unused-expression + wrapper.ref && wrapper.ref.updateFormMap$(formMap, wrapper.xFormWrapper.formData, true); } }; @@ -157,8 +157,8 @@ export const showModifyModal = (record: IUploadFile) => { export const showConfigureModal = async (record?: IConfigure) => { if (record) { - const result:any = await format2json(record.configValue); - record.configValue = result.result; + const result: any = await format2json(record.configValue); + record.configValue = result.result || record.configValue; } const xFormModal = { formMap: [ @@ -193,10 +193,69 @@ export const showConfigureModal = async (record?: IConfigure) => { return admin.editConfigure(value).then(data => { notification.success({ message: '编辑配置成功' }); }); + } else { + return admin.addNewConfigure(value).then(data => { + notification.success({ message: '新建配置成功' }); + }); + } + }, + }; + wrapper.open(xFormModal); +}; + +export const showConfigGatewayModal = async (record?: IConfigGateway) => { + const xFormModal = { + formMap: [ + { + key: 'type', + label: '配置类型', + rules: [{ required: true, message: '请选择配置类型' }], + type: "select", + options: admin.gatewayType.map((item: any, index: number) => ({ + key: index, + label: item.configName, + value: item.configType, + })), + attrs: { + disabled: record ? true : false, + } + }, { + key: 'name', + label: '配置键', + rules: [{ required: true, message: '请输入配置键' }], + attrs: { + disabled: record ? true : false, + }, + }, { + key: 'value', + label: '配置值', + type: 'text_area', + rules: [{ + required: true, + message: '请输入配置值', + }], + }, { + key: 'description', + label: '描述', + type: 'text_area', + rules: [{ required: true, message: '请输入备注' }], + }, + ], + formData: record || {}, + visible: true, + isWaitting: true, + title: `${record ? '编辑配置' : '新建配置'}`, + onSubmit: async (parmas: IConfigGateway) => { + if (record) { + parmas.id = record.id; + return admin.editConfigGateway(parmas).then(data => { + notification.success({ message: '编辑配置成功' }); + }); + } else { + return admin.addNewConfigGateway(parmas).then(data => { + notification.success({ message: '新建配置成功' }); + }); } - return admin.addNewConfigure(value).then(data => { - notification.success({ message: '新建配置成功' }); - }); }, }; wrapper.open(xFormModal); diff --git a/kafka-manager-console/src/container/modal/app.tsx b/kafka-manager-console/src/container/modal/app.tsx index bda37418..bb0320ec 100644 --- a/kafka-manager-console/src/container/modal/app.tsx +++ b/kafka-manager-console/src/container/modal/app.tsx @@ -85,7 +85,7 @@ export const showEditModal = (record?: IAppItem, from?: string, isDisabled?: boo ], formData: record, visible: true, - title: isDisabled ? '详情' : record ? '编辑' :
    应用申请应用申请文档
    , + title: isDisabled ? '详情' : record ? '编辑' :
    应用申请资源申请文档
    , // customRenderElement: isDisabled ? '' : record ? '' : 集群资源充足时，预计1分钟自动审批通过, isWaitting: true, onSubmit: (value: IAppItem) => { diff --git a/kafka-manager-console/src/container/modal/expert.tsx b/kafka-manager-console/src/container/modal/expert.tsx index 96a1f312..2ff5e5f2 100644 --- a/kafka-manager-console/src/container/modal/expert.tsx +++ b/kafka-manager-console/src/container/modal/expert.tsx @@ -20,14 +20,14 @@ export interface IRenderData { } export const migrationModal = (renderData: IRenderData[]) => { - const xFormWrapper = { + const xFormWrapper = { type: 'drawer', visible: true, width: 1000, title: '新建迁移任务', - customRenderElement: , + customRenderElement: , nofooter: true, noform: true, }; - wrapper.open(xFormWrapper as IXFormWrapper); + wrapper.open(xFormWrapper as IXFormWrapper); }; diff --git a/kafka-manager-console/src/container/modal/order.tsx b/kafka-manager-console/src/container/modal/order.tsx index 2930e3df..c982db4c 100644 --- a/kafka-manager-console/src/container/modal/order.tsx +++ b/kafka-manager-console/src/container/modal/order.tsx @@ -75,8 +75,8 @@ export const showApprovalModal = (info: IOrderInfo, status: number, from?: strin // }], rules: [{ required: true, - message: '请输入大于12小于999的整数', - pattern: /^([1-9]{1}[0-9]{2})$|^([2-9]{1}[0-9]{1})$|^(1[2-9]{1})$/, + message: '请输入大于0小于10000的整数', + pattern: /^\+?[1-9]\d{0,3}(\.\d*)?$/, }], }, { key: 'species', diff --git a/kafka-manager-console/src/container/modal/topic.tsx b/kafka-manager-console/src/container/modal/topic.tsx index 7053a497..d4df0318 100644 --- a/kafka-manager-console/src/container/modal/topic.tsx +++ b/kafka-manager-console/src/container/modal/topic.tsx @@ -88,7 +88,7 @@ export const applyTopic = () => { ], formData: {}, visible: true, - title: '申请Topic', + title: , okText: '确认', // customRenderElement: 集群资源充足时，预计1分钟自动审批通过, isWaitting: true, diff --git a/kafka-manager-console/src/container/search-filter.tsx b/kafka-manager-console/src/container/search-filter.tsx index 12603d40..ac5d6bc1 100644 --- a/kafka-manager-console/src/container/search-filter.tsx +++ b/kafka-manager-console/src/container/search-filter.tsx @@ -126,7 +126,7 @@ export class SearchAndFilterContainer extends React.Component diff --git a/kafka-manager-console/src/container/topic/topic-detail/connect-information.tsx b/kafka-manager-console/src/container/topic/topic-detail/connect-information.tsx index f323310c..1e5ab182 100644 --- a/kafka-manager-console/src/container/topic/topic-detail/connect-information.tsx +++ b/kafka-manager-console/src/container/topic/topic-detail/connect-information.tsx @@ -101,7 +101,9 @@ export class ConnectInformation extends SearchAndFilterContainer { <>
      -
    • 连接信息
      • +
    • + 连接信息 展示近20分钟的连接信息 +
      • {this.renderSearch('', '请输入连接信息', 'searchKey')}
    {this.renderConnectionInfo(this.getData(topic.connectionInfo))} diff --git a/kafka-manager-console/src/container/topic/topic-detail/group-id.tsx b/kafka-manager-console/src/container/topic/topic-detail/group-id.tsx index 20b7642f..b173ac41 100644 --- a/kafka-manager-console/src/container/topic/topic-detail/group-id.tsx +++ b/kafka-manager-console/src/container/topic/topic-detail/group-id.tsx @@ -138,7 +138,7 @@ export class GroupID extends SearchAndFilterContainer { public renderConsumerDetails() { const consumerGroup = this.consumerGroup; - const columns = [{ + const columns: any = [{ title: 'Partition ID', dataIndex: 'partitionId', key: 'partitionId', @@ -179,7 +179,8 @@ export class GroupID extends SearchAndFilterContainer { <>
    {consumerGroup} -
    +
    + {this.renderSearch('', '请输入Consumer ID')} @@ -187,7 +188,7 @@ export class GroupID extends SearchAndFilterContainer {
    @@ -214,7 +215,12 @@ export class GroupID extends SearchAndFilterContainer { dataIndex: 'location', key: 'location', width: '34%', - }, + }, { + title: '状态', + dataIndex: 'state', + key: 'state', + width: '34%', + } ]; return ( <> @@ -236,7 +242,17 @@ export class GroupID extends SearchAndFilterContainer { data = searchKey ? origin.filter((item: IConsumerGroups) => (item.consumerGroup !== undefined && item.consumerGroup !== null) && item.consumerGroup.toLowerCase().includes(searchKey as string), - ) : origin ; + ) : origin; + return data; + } + + public getDetailData(origin: T[]) { + let data: T[] = origin; + let { searchKey } = this.state; + searchKey = (searchKey + '').trim().toLowerCase(); + data = searchKey ? origin.filter((item: IConsumeDetails) => + (item.clientId !== undefined && item.clientId !== null) && item.clientId.toLowerCase().includes(searchKey as string), + ) : origin; return data; } diff --git a/kafka-manager-console/src/container/topic/topic-detail/reset-offset.tsx b/kafka-manager-console/src/container/topic/topic-detail/reset-offset.tsx index be0767e8..531f69c6 100644 --- a/kafka-manager-console/src/container/topic/topic-detail/reset-offset.tsx +++ b/kafka-manager-console/src/container/topic/topic-detail/reset-offset.tsx @@ -71,32 +71,32 @@ class ResetOffset extends React.Component { const { getFieldDecorator } = this.props.form; const { typeValue, offsetValue } = this.state; return ( - <> - - - + <> + + + {/* */}
    - + 重置到指定时间
    - - 最新offset - 自定义 - + + 最新offset + 自定义 + {typeValue === 'time' && offsetValue === 'custom' && getFieldDecorator('timestamp', { rules: [{ required: false, message: '' }], initialValue: moment(), - })( + })( { 重置指定分区及偏移 - + diff --git a/kafka-manager-console/src/lib/api.ts b/kafka-manager-console/src/lib/api.ts index f53f6852..8716d4ea 100644 --- a/kafka-manager-console/src/lib/api.ts +++ b/kafka-manager-console/src/lib/api.ts @@ -1,5 +1,5 @@ import fetch, { formFetch } from './fetch'; -import { IUploadFile, IUser, IQuotaModelItem, ILimitsItem, ITopic, IOrderParams, ISample, IMigration, IExecute, IEepand, IUtils, ITopicMetriceParams, IRegister, IEditTopic, IExpand, IDeleteTopic, INewRegions, INewLogical, IRebalance, INewBulidEnums, ITrigger, IApprovalOrder, IMonitorSilences, IConfigure, IBatchApproval } from 'types/base-type'; +import { IUploadFile, IUser, IQuotaModelItem, ILimitsItem, ITopic, IOrderParams, ISample, IMigration, IExecute, IEepand, IUtils, ITopicMetriceParams, IRegister, IEditTopic, IExpand, IDeleteTopic, INewRegions, INewLogical, IRebalance, INewBulidEnums, ITrigger, IApprovalOrder, IMonitorSilences, IConfigure, IConfigGateway, IBatchApproval } from 'types/base-type'; import { IRequestParams } from 'types/alarm'; import { apiCache } from 'lib/api-cache'; @@ -442,6 +442,34 @@ export const deleteConfigure = (configKey: string) => { }); }; +export const getGatewayList = () => { + return fetch(`/rd/gateway-configs`); +}; + +export const getGatewayType = () => { + return fetch(`/op/gateway-configs/type-enums`); +}; + +export const addNewConfigGateway = (params: IConfigGateway) => { + return fetch(`/op/gateway-configs`, { + method: 'POST', + body: JSON.stringify(params), + }); +}; + +export const editConfigGateway = (params: IConfigGateway) => { + return fetch(`/op/gateway-configs`, { + method: 'PUT', + body: JSON.stringify(params), + }); +}; +export const deleteConfigGateway = (params: IConfigure) => { + return fetch(`/op/gateway-configs`, { + method: 'DELETE', + body: JSON.stringify(params), + }); +}; + export const getDataCenter = () => { return fetch(`/normal/configs/idc`); }; @@ -530,6 +558,23 @@ export const getControllerHistory = (clusterId: number) => { return fetch(`/rd/clusters/${clusterId}/controller-history`); }; +export const getCandidateController = (clusterId: number) => { + return fetch(`/rd/clusters/${clusterId}/controller-preferred-candidates`); +}; + +export const addCandidateController = (params:any) => { + return fetch(`/op/cluster-controller/preferred-candidates`, { + method: 'POST', + body: JSON.stringify(params), + }); +}; + +export const deleteCandidateCancel = (params:any)=>{ + return fetch(`/op/cluster-controller/preferred-candidates`, { + method: 'DELETE', + body: JSON.stringify(params), + }); +} /** * 运维管控 broker */ diff --git a/kafka-manager-console/src/lib/line-charts-config.ts b/kafka-manager-console/src/lib/line-charts-config.ts index fe9880a6..4a667c0c 100644 --- a/kafka-manager-console/src/lib/line-charts-config.ts +++ b/kafka-manager-console/src/lib/line-charts-config.ts @@ -77,7 +77,7 @@ export const getControlMetricOption = (type: IOptionType, data: IClusterMetrics[ name = '条'; data.map(item => { item.messagesInPerSec = item.messagesInPerSec !== null ? Number(item.messagesInPerSec.toFixed(2)) : null; - }); + }); break; case 'brokerNum': case 'topicNum': @@ -224,7 +224,7 @@ export const getClusterMetricOption = (type: IOptionType, record: IClusterMetric name = '条'; data.map(item => { item.messagesInPerSec = item.messagesInPerSec !== null ? Number(item.messagesInPerSec.toFixed(2)) : null; - }); + }); break; default: const { name: unitName, data: xData } = dealFlowData(metricTypeMap[type], data); @@ -248,8 +248,8 @@ export const getClusterMetricOption = (type: IOptionType, record: IClusterMetric const unitSeries = item.data[item.seriesName] !== null ? Number(item.data[item.seriesName]) : null; // tslint:disable-next-line:max-line-length result += ''; - if ( (item.data.produceThrottled && item.seriesName === 'appIdBytesInPerSec') - || (item.data.consumeThrottled && item.seriesName === 'appIdBytesOutPerSec') ) { + if ((item.data.produceThrottled && item.seriesName === 'appIdBytesInPerSec') + || (item.data.consumeThrottled && item.seriesName === 'appIdBytesOutPerSec')) { return result += item.seriesName + ': ' + unitSeries + '（被限流）' + '
    '; } return result += item.seriesName + ': ' + unitSeries + '
    '; @@ -317,7 +317,7 @@ export const getMonitorMetricOption = (seriesName: string, data: IMetricPoint[]) if (ele.name === item.seriesName) { // tslint:disable-next-line:max-line-length result += ''; - return result += item.seriesName + ': ' + (item.data.value === null ? '' : item.data.value.toFixed(2)) + '
    '; + return result += item.seriesName + ': ' + (item.data.value === null ? '' : item.data.value.toFixed(2)) + '
    '; } }); }); diff --git a/kafka-manager-console/src/store/admin-monitor.ts b/kafka-manager-console/src/store/admin-monitor.ts index 4071e1c5..7e257637 100644 --- a/kafka-manager-console/src/store/admin-monitor.ts +++ b/kafka-manager-console/src/store/admin-monitor.ts @@ -3,6 +3,11 @@ import { observable, action } from 'mobx'; import { getBrokersMetricsHistory } from 'lib/api'; import { IClusterMetrics } from 'types/base-type'; +const STATUS = { + PENDING: 'pending', + REJECT: 'reject', + FULLFILLED: 'fullfilled' +} class AdminMonitor { @observable public currentClusterId = null as number; @@ -33,33 +38,42 @@ class AdminMonitor { @action.bound public setBrokersChartsData(data: IClusterMetrics[]) { this.brokersMetricsHistory = data; - this.setRequestId(null); + this.setRequestId(STATUS.FULLFILLED); + Promise.all(this.taskQueue).then(() => { + this.setRequestId(null); + this.taskQueue = []; + }) return data; } + public taskQueue = [] as any[]; public getBrokersMetricsList = async (startTime: string, endTime: string) => { - if (this.requestId && this.requestId !== 'error') { - return new Promise((res, rej) => { - window.setTimeout(() => { - if (this.requestId === 'error') { - rej(); - } else { + if (this.requestId) { + //逐条定时查询任务状态 + const p = new Promise((res, rej) => { + const timer = window.setInterval(() => { + if (this.requestId === STATUS.REJECT) { + rej(this.brokersMetricsHistory); + window.clearInterval(timer); + } else if (this.requestId === STATUS.FULLFILLED) { res(this.brokersMetricsHistory); + window.clearInterval(timer); } - }, 800); // TODO: 该实现方式待优化 + }, (this.taskQueue.length + 1) * 100); }); + this.taskQueue.push(p); + return p; } - this.setRequestId('requesting'); + this.setRequestId(STATUS.PENDING); return getBrokersMetricsHistory(this.currentClusterId, this.currentBrokerId, startTime, endTime) - .then(this.setBrokersChartsData).catch(() => this.setRequestId('error')); + .then(this.setBrokersChartsData).catch(() => this.setRequestId(STATUS.REJECT)); } public getBrokersChartsData = async (startTime: string, endTime: string, reload?: boolean) => { if (this.brokersMetricsHistory && !reload) { return new Promise(res => res(this.brokersMetricsHistory)); } - return this.getBrokersMetricsList(startTime, endTime); } } diff --git a/kafka-manager-console/src/store/admin.ts b/kafka-manager-console/src/store/admin.ts index f3d08264..bd641773 100644 --- a/kafka-manager-console/src/store/admin.ts +++ b/kafka-manager-console/src/store/admin.ts @@ -1,5 +1,5 @@ import { observable, action } from 'mobx'; -import { INewBulidEnums, ILabelValue, IClusterReal, IOptionType, IClusterMetrics, IClusterTopics, IKafkaFiles, IMetaData, IConfigure, IBrokerData, IOffset, IController, IBrokersBasicInfo, IBrokersStatus, IBrokersTopics, IBrokersPartitions, IBrokersAnalysis, IAnalysisTopicVO, IBrokersMetadata, IBrokersRegions, IThrottles, ILogicalCluster, INewRegions, INewLogical, ITaskManage, IPartitionsLocation, ITaskType, ITasksEnums, ITasksMetaData, ITaskStatusDetails, IKafkaRoles, IEnumsMap, IStaffSummary, IBill, IBillDetail } from 'types/base-type'; +import { INewBulidEnums, ILabelValue, IClusterReal, IOptionType, IClusterMetrics, IClusterTopics, IKafkaFiles, IMetaData, IConfigure, IConfigGateway, IBrokerData, IOffset, IController, IBrokersBasicInfo, IBrokersStatus, IBrokersTopics, IBrokersPartitions, IBrokersAnalysis, IAnalysisTopicVO, IBrokersMetadata, IBrokersRegions, IThrottles, ILogicalCluster, INewRegions, INewLogical, ITaskManage, IPartitionsLocation, ITaskType, ITasksEnums, ITasksMetaData, ITaskStatusDetails, IKafkaRoles, IEnumsMap, IStaffSummary, IBill, IBillDetail } from 'types/base-type'; import { deleteCluster, getBasicInfo, @@ -12,7 +12,12 @@ import { getConfigure, addNewConfigure, editConfigure, + addNewConfigGateway, deleteConfigure, + getGatewayList, + getGatewayType, + editConfigGateway, + deleteConfigGateway, getDataCenter, getClusterBroker, getClusterConsumer, @@ -49,6 +54,9 @@ import { getStaffSummary, getBillStaffSummary, getBillStaffDetail, + getCandidateController, + addCandidateController, + deleteCandidateCancel } from 'lib/api'; import { getControlMetricOption, getClusterMetricOption } from 'lib/line-charts-config'; @@ -59,6 +67,7 @@ import { transBToMB } from 'lib/utils'; import moment from 'moment'; import { timestore } from './time'; +import { message } from 'component/antd'; class Admin { @observable @@ -97,6 +106,12 @@ class Admin { @observable public configureList: IConfigure[] = []; + @observable + public configGatewayList: IConfigGateway[] = []; + + @observable + public gatewayType: []; + @observable public dataCenterList: string[] = []; @@ -142,6 +157,12 @@ class Admin { @observable public controllerHistory: IController[] = []; + @observable + public controllerCandidate: IController[] = []; + + @observable + public filtercontrollerCandidate: string = ''; + @observable public brokersPartitions: IBrokersPartitions[] = []; @@ -152,7 +173,7 @@ class Admin { public brokersAnalysisTopic: IAnalysisTopicVO[] = []; @observable - public brokersMetadata: IBrokersMetadata[] = []; + public brokersMetadata: IBrokersMetadata[] | any = []; @observable public brokersRegions: IBrokersRegions[] = []; @@ -206,10 +227,10 @@ class Admin { public kafkaRoles: IKafkaRoles[]; @observable - public controlType: IOptionType = 'byteIn/byteOut' ; + public controlType: IOptionType = 'byteIn/byteOut'; @observable - public type: IOptionType = 'byteIn/byteOut' ; + public type: IOptionType = 'byteIn/byteOut'; @observable public currentClusterId = null as number; @@ -241,7 +262,7 @@ class Admin { @action.bound public setClusterRealTime(data: IClusterReal) { - this.clusterRealData = data; + this.clusterRealData = data; this.getRealClusterLoading(false); } @@ -284,7 +305,7 @@ class Admin { return { ...item, label: item.fileName, - value: item.fileName + ',' + item.fileMd5, + value: item.fileName + ',' + item.fileMd5, }; })); } @@ -306,6 +327,20 @@ class Admin { }) : []; } + @action.bound + public setConfigGatewayList(data: IConfigGateway[]) { + this.configGatewayList = data ? data.map((item, index) => { + item.key = index; + return item; + }) : []; + } + + @action.bound + public setConfigGatewayType(data: any) { + this.setLoading(false); + this.gatewayType = data || []; + } + @action.bound public setDataCenter(data: string[]) { this.dataCenterList = data || []; @@ -335,6 +370,17 @@ class Admin { }) : []; } + @action.bound + public setCandidateController(data: IController[]) { + this.controllerCandidate = data ? data.map((item, index) => { + item.key = index; + return item; + }) : []; + this.filtercontrollerCandidate = data?data.map((item,index)=>{ + return item.brokerId + }).join(','):'' + } + @action.bound public setBrokersBasicInfo(data: IBrokersBasicInfo) { this.brokersBasicInfo = data; @@ -356,10 +402,10 @@ class Admin { this.replicaStatus = data.brokerReplicaStatusList.slice(1); this.bytesInStatus.forEach((item, index) => { - this.peakValueList.push({ name: peakValueMap[index], value: item}); + this.peakValueList.push({ name: peakValueMap[index], value: item }); }); this.replicaStatus.forEach((item, index) => { - this.copyValueList.push({name: copyValueMap[index], value: item}); + this.copyValueList.push({ name: copyValueMap[index], value: item }); }); } @@ -415,16 +461,16 @@ class Admin { } @action.bound - public setBrokersMetadata(data: IBrokersMetadata[]) { - this.brokersMetadata = data ? data.map((item, index) => { - item.key = index; - return { - ...item, - text: `${item.host} （BrokerID：${item.brokerId}）`, - label: item.host, - value: item.brokerId, - }; - }) : []; + public setBrokersMetadata(data: IBrokersMetadata[]|any) { + this.brokersMetadata = data ? data.map((item:any, index:any) => { + item.key = index; + return { + ...item, + text: `${item.host} （BrokerID：${item.brokerId}）`, + label: item.host, + value: item.brokerId, + }; + }) : []; } @action.bound @@ -461,9 +507,9 @@ class Admin { @action.bound public setLogicalClusters(data: ILogicalCluster[]) { this.logicalClusters = data ? data.map((item, index) => { - item.key = index; - return item; - }) : []; + item.key = index; + return item; + }) : []; } @action.bound @@ -474,25 +520,25 @@ class Admin { @action.bound public setClustersThrottles(data: IThrottles[]) { this.clustersThrottles = data ? data.map((item, index) => { - item.key = index; - return item; - }) : []; + item.key = index; + return item; + }) : []; } @action.bound public setPartitionsLocation(data: IPartitionsLocation[]) { this.partitionsLocation = data ? data.map((item, index) => { - item.key = index; - return item; - }) : []; + item.key = index; + return item; + }) : []; } @action.bound public setTaskManagement(data: ITaskManage[]) { this.taskManagement = data ? data.map((item, index) => { - item.key = index; - return item; - }) : []; + item.key = index; + return item; + }) : []; } @action.bound @@ -568,7 +614,7 @@ class Admin { return deleteCluster(clusterId).then(() => this.getMetaData(true)); } - public getPeakFlowChartData(value: ILabelValue[], map: string []) { + public getPeakFlowChartData(value: ILabelValue[], map: string[]) { return getPieChartOption(value, map); } @@ -627,6 +673,30 @@ class Admin { deleteConfigure(configKey).then(() => this.getConfigure()); } + public getGatewayList() { + getGatewayList().then(this.setConfigGatewayList); + } + + public getGatewayType() { + this.setLoading(true); + getGatewayType().then(this.setConfigGatewayType); + } + + public addNewConfigGateway(params: IConfigGateway) { + return addNewConfigGateway(params).then(() => this.getGatewayList()); + } + + public editConfigGateway(params: IConfigGateway) { + return editConfigGateway(params).then(() => this.getGatewayList()); + } + + public deleteConfigGateway(params: any) { + deleteConfigGateway(params).then(() => { + // message.success('删除成功') + this.getGatewayList() + }); + } + public getDataCenter() { getDataCenter().then(this.setDataCenter); } @@ -643,6 +713,20 @@ class Admin { return getControllerHistory(clusterId).then(this.setControllerHistory); } + public getCandidateController(clusterId: number) { + return getCandidateController(clusterId).then(data=>{ + return this.setCandidateController(data) + }); + } + + public addCandidateController(clusterId: number, brokerIdList: any) { + return addCandidateController({clusterId, brokerIdList}).then(()=>this.getCandidateController(clusterId)); + } + + public deleteCandidateCancel(clusterId: number, brokerIdList: any){ + return deleteCandidateCancel({clusterId, brokerIdList}).then(()=>this.getCandidateController(clusterId)); + } + public getBrokersBasicInfo(clusterId: number, brokerId: number) { return getBrokersBasicInfo(clusterId, brokerId).then(this.setBrokersBasicInfo); } diff --git a/kafka-manager-console/src/store/alarm.ts b/kafka-manager-console/src/store/alarm.ts index b3e004df..e57631f0 100644 --- a/kafka-manager-console/src/store/alarm.ts +++ b/kafka-manager-console/src/store/alarm.ts @@ -181,6 +181,7 @@ class Alarm { public modifyMonitorStrategy(params: IRequestParams) { return modifyMonitorStrategy(params).then(() => { message.success('操作成功'); + window.location.href = `${urlPrefix}/alarm`; }).finally(() => this.setLoading(false)); } diff --git a/kafka-manager-console/src/store/users.ts b/kafka-manager-console/src/store/users.ts index 8d53114e..249a0187 100644 --- a/kafka-manager-console/src/store/users.ts +++ b/kafka-manager-console/src/store/users.ts @@ -19,6 +19,9 @@ export class Users { @observable public staff: IStaff[] = []; + @observable + public newPassWord: any = null; + @action.bound public setAccount(data: IUser) { setCookie([{ key: 'role', value: `${data.role}`, time: 1 }]); @@ -42,6 +45,11 @@ export class Users { this.loading = value; } + @action.bound + public setNewPassWord(value: boolean) { + this.newPassWord = value; + } + public getAccount() { getAccount().then(this.setAccount); } diff --git a/kafka-manager-console/src/types/base-type.ts b/kafka-manager-console/src/types/base-type.ts index 1170a696..605fd4fc 100644 --- a/kafka-manager-console/src/types/base-type.ts +++ b/kafka-manager-console/src/types/base-type.ts @@ -190,6 +190,7 @@ export interface IUser { chineseName?: string; department?: string; key?: number; + confirmPassword?:string } export interface IOffset { @@ -486,6 +487,17 @@ export interface IConfigure { key?: number; } +export interface IConfigGateway { + id: number; + key?: number; + modifyTime: number; + name: string; + value: string; + version: string; + type: string; + description: string; +} + export interface IEepand { brokerIdList: number[]; clusterId: number; @@ -650,8 +662,10 @@ export interface IBrokerData { export interface IController { brokerId: number; host: string; - timestamp: number; - version: number; + timestamp?: number; + version?: number; + startTime?: number; + status?: number; key?: number; } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java index 59453919..e3b8f23f 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java @@ -15,10 +15,7 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl; import com.xiaojukeji.kafka.manager.dao.ControllerDao; import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConnectorWrap; -import com.xiaojukeji.kafka.manager.dao.TopicDao; -import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.service.JmxService; -import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils; import com.xiaojukeji.kafka.manager.service.zookeeper.*; import com.xiaojukeji.kafka.manager.service.service.ClusterService; import com.xiaojukeji.kafka.manager.common.zookeeper.ZkPathUtil; @@ -49,15 +46,6 @@ public class PhysicalClusterMetadataManager { @Autowired private ClusterService clusterService; - @Autowired - private ConfigUtils configUtils; - - @Autowired - private TopicDao topicDao; - - @Autowired - private AuthorityDao authorityDao; - private final static Map CLUSTER_MAP = new ConcurrentHashMap<>(); private final static Map CONTROLLER_DATA_MAP = new ConcurrentHashMap<>(); @@ -133,7 +121,7 @@ public class PhysicalClusterMetadataManager { zkConfig.watchChildren(ZkPathUtil.BROKER_IDS_ROOT, brokerListener); //增加Topic监控 - TopicStateListener topicListener = new TopicStateListener(clusterDO.getId(), zkConfig, topicDao, authorityDao); + TopicStateListener topicListener = new TopicStateListener(clusterDO.getId(), zkConfig); topicListener.init(); zkConfig.watchChildren(ZkPathUtil.BROKER_TOPICS_ROOT, topicListener); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java index 004a3f51..2feb321b 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java @@ -4,6 +4,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.ClusterDetailDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.ControllerPreferredCandidate; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.ControllerPreferredCandidateDTO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.ClusterNameDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterMetricsDO; @@ -43,7 +44,7 @@ public interface ClusterService { ClusterNameDTO getClusterName(Long logicClusterId); - ResultStatus deleteById(Long clusterId); + ResultStatus deleteById(Long clusterId, String operator); /** * 获取优先被选举为controller的broker @@ -51,4 +52,20 @@ public interface ClusterService { * @return void */ Result> getControllerPreferredCandidates(Long clusterId); + + /** + * 增加优先被选举为controller的broker + * @param clusterId 集群ID + * @param brokerIdList brokerId列表 + * @return + */ + Result addControllerPreferredCandidates(Long clusterId, List brokerIdList); + + /** + * 减少优先被选举为controller的broker + * @param clusterId 集群ID + * @param brokerIdList brokerId列表 + * @return + */ + Result deleteControllerPreferredCandidates(Long clusterId, List brokerIdList); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/OperateRecordService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/OperateRecordService.java index c5007ac6..5b2909ca 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/OperateRecordService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/OperateRecordService.java @@ -1,9 +1,12 @@ package com.xiaojukeji.kafka.manager.service.service; +import com.xiaojukeji.kafka.manager.common.bizenum.ModuleEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.OperateEnum; import com.xiaojukeji.kafka.manager.common.entity.dto.rd.OperateRecordDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.OperateRecordDO; import java.util.List; +import java.util.Map; /** * @author zhongyuankai @@ -12,5 +15,7 @@ import java.util.List; public interface OperateRecordService { int insert(OperateRecordDO operateRecordDO); + int insert(String operator, ModuleEnum module, String resourceName, OperateEnum operate, Map content); + List queryByCondt(OperateRecordDTO dto); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java index d24b2d24..d52d3bc7 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java @@ -26,4 +26,20 @@ public interface ZookeeperService { * @return 操作结果 */ Result> getControllerPreferredCandidates(Long clusterId); + + /** + * 增加优先被选举为controller的broker + * @param clusterId 集群ID + * @param brokerId brokerId + * @return + */ + Result addControllerPreferredCandidate(Long clusterId, Integer brokerId); + + /** + * 减少优先被选举为controller的broker + * @param clusterId 集群ID + * @param brokerId brokerId + * @return + */ + Result deleteControllerPreferredCandidate(Long clusterId, Integer brokerId); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AppService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AppService.java index c78946b6..82aa5513 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AppService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AppService.java @@ -17,7 +17,7 @@ public interface AppService { * @param appDO appDO * @return int */ - ResultStatus addApp(AppDO appDO); + ResultStatus addApp(AppDO appDO, String operator); /** * 删除数据 diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java index 09b4a071..200b3cf4 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java @@ -60,10 +60,8 @@ public class AppServiceImpl implements AppService { @Autowired private OperateRecordService operateRecordService; - - @Override - public ResultStatus addApp(AppDO appDO) { + public ResultStatus addApp(AppDO appDO, String operator) { try { if (appDao.insert(appDO) < 1) { LOGGER.warn("class=AppServiceImpl||method=addApp||AppDO={}||msg=add fail,{}",appDO,ResultStatus.MYSQL_ERROR.getMessage()); @@ -75,6 +73,15 @@ public class AppServiceImpl implements AppService { kafkaUserDO.setOperation(OperationStatusEnum.CREATE.getCode()); kafkaUserDO.setUserType(0); kafkaUserDao.insert(kafkaUserDO); + + Map content = new HashMap<>(); + content.put("appId", appDO.getAppId()); + content.put("name", appDO.getName()); + content.put("applicant", appDO.getApplicant()); + content.put("password", appDO.getPassword()); + content.put("principals", appDO.getPrincipals()); + content.put("description", appDO.getDescription()); + operateRecordService.insert(operator, ModuleEnum.APP, appDO.getName(), OperateEnum.ADD, content); } catch (DuplicateKeyException e) { LOGGER.error("class=AppServiceImpl||method=addApp||errMsg={}||appDO={}|", e.getMessage(), appDO, e); return ResultStatus.RESOURCE_ALREADY_EXISTED; @@ -141,6 +148,12 @@ public class AppServiceImpl implements AppService { appDO.setDescription(dto.getDescription()); if (appDao.updateById(appDO) > 0) { + Map content = new HashMap<>(); + content.put("appId", appDO.getAppId()); + content.put("name", appDO.getName()); + content.put("principals", appDO.getPrincipals()); + content.put("description", appDO.getDescription()); + operateRecordService.insert(operator, ModuleEnum.APP, appDO.getName(), OperateEnum.EDIT, content); return ResultStatus.SUCCESS; } } catch (DuplicateKeyException e) { diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java index fce7b605..18ee0a0d 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java @@ -221,13 +221,24 @@ public class GatewayConfigServiceImpl implements GatewayConfigService { if (ValidateUtils.isNull(oldGatewayConfigDO)) { return Result.buildFrom(ResultStatus.RESOURCE_NOT_EXIST); } + if (!oldGatewayConfigDO.getName().equals(newGatewayConfigDO.getName()) || !oldGatewayConfigDO.getType().equals(newGatewayConfigDO.getType()) || ValidateUtils.isBlank(newGatewayConfigDO.getValue())) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } - newGatewayConfigDO.setVersion(oldGatewayConfigDO.getVersion() + 1); - if (gatewayConfigDao.updateById(oldGatewayConfigDO) > 0) { + + // 获取当前同类配置, 插入之后需要增大这个version + List gatewayConfigDOList = gatewayConfigDao.getByConfigType(newGatewayConfigDO.getType()); + Long version = 1L; + for (GatewayConfigDO elem: gatewayConfigDOList) { + if (elem.getVersion() > version) { + version = elem.getVersion() + 1L; + } + } + + newGatewayConfigDO.setVersion(version); + if (gatewayConfigDao.updateById(newGatewayConfigDO) > 0) { return Result.buildSuc(); } return Result.buildFrom(ResultStatus.MYSQL_ERROR); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java index fd28308d..e1a619a8 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java @@ -1,6 +1,8 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.DBStatusEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.ModuleEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.OperateEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.ClusterDetailDTO; @@ -15,10 +17,7 @@ import com.xiaojukeji.kafka.manager.dao.ClusterMetricsDao; import com.xiaojukeji.kafka.manager.dao.ControllerDao; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; -import com.xiaojukeji.kafka.manager.service.service.ClusterService; -import com.xiaojukeji.kafka.manager.service.service.ConsumerService; -import com.xiaojukeji.kafka.manager.service.service.RegionService; -import com.xiaojukeji.kafka.manager.service.service.ZookeeperService; +import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; @@ -65,6 +64,9 @@ public class ClusterServiceImpl implements ClusterService { @Autowired private ZookeeperService zookeeperService; + @Autowired + private OperateRecordService operateRecordService; + @Override public ResultStatus addNew(ClusterDO clusterDO, String operator) { if (ValidateUtils.isNull(clusterDO) || ValidateUtils.isNull(operator)) { @@ -74,6 +76,12 @@ public class ClusterServiceImpl implements ClusterService { return ResultStatus.ZOOKEEPER_CONNECT_FAILED; } try { + Map content = new HashMap<>(); + content.put("zk address", clusterDO.getZookeeper()); + content.put("bootstrap servers", clusterDO.getBootstrapServers()); + content.put("security properties", clusterDO.getSecurityProperties()); + content.put("jmx properties", clusterDO.getJmxProperties()); + operateRecordService.insert(operator, ModuleEnum.CLUSTER, clusterDO.getClusterName(), OperateEnum.ADD, content); if (clusterDao.insert(clusterDO) <= 0) { LOGGER.error("add new cluster failed, clusterDO:{}.", clusterDO); return ResultStatus.MYSQL_ERROR; @@ -103,6 +111,12 @@ public class ClusterServiceImpl implements ClusterService { // 不允许修改zk地址 return ResultStatus.CHANGE_ZOOKEEPER_FORBIDDEN; } + Map content = new HashMap<>(); + content.put("cluster id", clusterDO.getId().toString()); + content.put("security properties", clusterDO.getSecurityProperties()); + content.put("jmx properties", clusterDO.getJmxProperties()); + operateRecordService.insert(operator, ModuleEnum.CLUSTER, clusterDO.getClusterName(), OperateEnum.EDIT, content); + clusterDO.setStatus(originClusterDO.getStatus()); return updateById(clusterDO); } @@ -201,7 +215,7 @@ public class ClusterServiceImpl implements ClusterService { if (zk != null) { zk.close(); } - } catch (Throwable t) { + } catch (Exception e) { return false; } } @@ -254,12 +268,15 @@ public class ClusterServiceImpl implements ClusterService { } @Override - public ResultStatus deleteById(Long clusterId) { + public ResultStatus deleteById(Long clusterId, String operator) { List regionDOList = regionService.getByClusterId(clusterId); if (!ValidateUtils.isEmptyList(regionDOList)) { return ResultStatus.OPERATION_FORBIDDEN; } try { + Map content = new HashMap<>(); + content.put("cluster id", clusterId.toString()); + operateRecordService.insert(operator, ModuleEnum.CLUSTER, String.valueOf(clusterId), OperateEnum.DELETE, content); if (clusterDao.deleteById(clusterId) <= 0) { LOGGER.error("delete cluster failed, clusterId:{}.", clusterId); return ResultStatus.MYSQL_ERROR; @@ -273,8 +290,9 @@ public class ClusterServiceImpl implements ClusterService { private ClusterDetailDTO getClusterDetailDTO(ClusterDO clusterDO, Boolean needDetail) { if (ValidateUtils.isNull(clusterDO)) { - return null; + return new ClusterDetailDTO(); } + ClusterDetailDTO dto = new ClusterDetailDTO(); dto.setClusterId(clusterDO.getId()); dto.setClusterName(clusterDO.getClusterName()); @@ -283,6 +301,7 @@ public class ClusterServiceImpl implements ClusterService { dto.setKafkaVersion(physicalClusterMetadataManager.getKafkaVersionFromCache(clusterDO.getId())); dto.setIdc(configUtils.getIdc()); dto.setSecurityProperties(clusterDO.getSecurityProperties()); + dto.setJmxProperties(clusterDO.getJmxProperties()); dto.setStatus(clusterDO.getStatus()); dto.setGmtCreate(clusterDO.getGmtCreate()); dto.setGmtModify(clusterDO.getGmtModify()); @@ -321,4 +340,39 @@ public class ClusterServiceImpl implements ClusterService { } return Result.buildSuc(controllerPreferredCandidateList); } + + @Override + public Result addControllerPreferredCandidates(Long clusterId, List brokerIdList) { + if (ValidateUtils.isNull(clusterId) || ValidateUtils.isEmptyList(brokerIdList)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + // 增加的BrokerId需要判断是否存活 + for (Integer brokerId: brokerIdList) { + if (!PhysicalClusterMetadataManager.isBrokerAlive(clusterId, brokerId)) { + return Result.buildFrom(ResultStatus.BROKER_NOT_EXIST); + } + + Result result = zookeeperService.addControllerPreferredCandidate(clusterId, brokerId); + if (result.failed()) { + return result; + } + } + return Result.buildSuc(); + } + + @Override + public Result deleteControllerPreferredCandidates(Long clusterId, List brokerIdList) { + if (ValidateUtils.isNull(clusterId) || ValidateUtils.isEmptyList(brokerIdList)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + for (Integer brokerId: brokerIdList) { + Result result = zookeeperService.deleteControllerPreferredCandidate(clusterId, brokerId); + if (result.failed()) { + return result; + } + } + return Result.buildSuc(); + } } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java index e228d36c..0d60d828 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java @@ -129,7 +129,7 @@ public class ConsumerServiceImpl implements ConsumerService { } summary.setState(consumerGroupSummary.state()); - java.util.Iterator> it = JavaConversions.asJavaIterator(consumerGroupSummary.consumers().iterator()); + Iterator> it = JavaConversions.asJavaIterator(consumerGroupSummary.consumers().iterator()); while (it.hasNext()) { List consumerSummaryList = JavaConversions.asJavaList(it.next()); for (AdminClient.ConsumerSummary consumerSummary: consumerSummaryList) { diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/OperateRecordServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/OperateRecordServiceImpl.java index 47702eaa..290bbae5 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/OperateRecordServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/OperateRecordServiceImpl.java @@ -1,7 +1,10 @@ package com.xiaojukeji.kafka.manager.service.service.impl; +import com.xiaojukeji.kafka.manager.common.bizenum.ModuleEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.OperateEnum; import com.xiaojukeji.kafka.manager.common.entity.dto.rd.OperateRecordDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.OperateRecordDO; +import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.dao.OperateRecordDao; import com.xiaojukeji.kafka.manager.service.service.OperateRecordService; @@ -10,6 +13,7 @@ import org.springframework.stereotype.Service; import java.util.Date; import java.util.List; +import java.util.Map; /** * @author zhongyuankai @@ -25,6 +29,17 @@ public class OperateRecordServiceImpl implements OperateRecordService { return operateRecordDao.insert(operateRecordDO); } + @Override + public int insert(String operator, ModuleEnum module, String resourceName, OperateEnum operate, Map content) { + OperateRecordDO operateRecordDO = new OperateRecordDO(); + operateRecordDO.setOperator(operator); + operateRecordDO.setModuleId(module.getCode()); + operateRecordDO.setResource(resourceName); + operateRecordDO.setOperateId(operate.getCode()); + operateRecordDO.setContent(JsonUtils.toJSONString(content)); + return insert(operateRecordDO); + } + @Override public List queryByCondt(OperateRecordDTO dto) { return operateRecordDao.queryByCondt( diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java index 0b42d068..6ee9a499 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java @@ -1,7 +1,10 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.KafkaClientEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.ModuleEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.OperateEnum; import com.xiaojukeji.kafka.manager.common.bizenum.TopicAuthorityEnum; +import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant; import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections; import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; @@ -80,6 +83,9 @@ public class TopicManagerServiceImpl implements TopicManagerService { @Autowired private RegionService regionService; + @Autowired + private OperateRecordService operateRecordService; + @Override public List listAll() { try { @@ -293,6 +299,10 @@ public class TopicManagerServiceImpl implements TopicManagerService { Map topicMap) { List dtoList = new ArrayList<>(); for (String topicName: PhysicalClusterMetadataManager.getTopicNameList(clusterDO.getId())) { + if (topicName.equals(KafkaConstant.COORDINATOR_TOPIC_NAME) || topicName.equals(KafkaConstant.TRANSACTION_TOPIC_NAME)) { + continue; + } + LogicalClusterDO logicalClusterDO = logicalClusterMetadataManager.getTopicLogicalCluster( clusterDO.getId(), topicName @@ -336,6 +346,12 @@ public class TopicManagerServiceImpl implements TopicManagerService { if (ValidateUtils.isNull(topicDO)) { return ResultStatus.TOPIC_NOT_EXIST; } + + Map content = new HashMap<>(2); + content.put("clusterId", clusterId); + content.put("topicName", topicName); + recordOperation(content, topicName, operator); + topicDO.setDescription(description); if (topicDao.updateByName(topicDO) > 0) { return ResultStatus.SUCCESS; @@ -359,6 +375,12 @@ public class TopicManagerServiceImpl implements TopicManagerService { return ResultStatus.APP_NOT_EXIST; } + Map content = new HashMap<>(4); + content.put("clusterId", clusterId); + content.put("topicName", topicName); + content.put("appId", appId); + recordOperation(content, topicName, operator); + TopicDO topicDO = topicDao.getByTopicName(clusterId, topicName); if (ValidateUtils.isNull(topicDO)) { // 不存在, 则需要插入 @@ -389,6 +411,16 @@ public class TopicManagerServiceImpl implements TopicManagerService { return ResultStatus.MYSQL_ERROR; } + private void recordOperation(Map content, String topicName, String operator) { + OperateRecordDO operateRecordDO = new OperateRecordDO(); + operateRecordDO.setModuleId(ModuleEnum.TOPIC.getCode()); + operateRecordDO.setOperateId(OperateEnum.EDIT.getCode()); + operateRecordDO.setResource(topicName); + operateRecordDO.setContent(JsonUtils.toJSONString(content)); + operateRecordDO.setOperator(operator); + operateRecordService.insert(operateRecordDO); + } + @Override public int deleteByTopicName(Long clusterId, String topicName) { try { diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java index aa31ed33..c4c89513 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java @@ -70,4 +70,58 @@ public class ZookeeperServiceImpl implements ZookeeperService { } return Result.buildFrom(ResultStatus.ZOOKEEPER_READ_FAILED); } + + @Override + public Result addControllerPreferredCandidate(Long clusterId, Integer brokerId) { + if (ValidateUtils.isNull(clusterId)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + ZkConfigImpl zkConfig = PhysicalClusterMetadataManager.getZKConfig(clusterId); + if (ValidateUtils.isNull(zkConfig)) { + return Result.buildFrom(ResultStatus.ZOOKEEPER_CONNECT_FAILED); + } + + try { + if (zkConfig.checkPathExists(ZkPathUtil.getControllerCandidatePath(brokerId))) { + // 节点已经存在, 则直接忽略 + return Result.buildSuc(); + } + + if (!zkConfig.checkPathExists(ZkPathUtil.D_CONFIG_EXTENSION_ROOT_NODE)) { + zkConfig.setOrCreatePersistentNodeStat(ZkPathUtil.D_CONFIG_EXTENSION_ROOT_NODE, ""); + } + + if (!zkConfig.checkPathExists(ZkPathUtil.D_CONTROLLER_CANDIDATES)) { + zkConfig.setOrCreatePersistentNodeStat(ZkPathUtil.D_CONTROLLER_CANDIDATES, ""); + } + + zkConfig.setOrCreatePersistentNodeStat(ZkPathUtil.getControllerCandidatePath(brokerId), ""); + return Result.buildSuc(); + } catch (Exception e) { + LOGGER.error("class=ZookeeperServiceImpl||method=addControllerPreferredCandidate||clusterId={}||brokerId={}||errMsg={}||", clusterId, brokerId, e.getMessage()); + } + return Result.buildFrom(ResultStatus.ZOOKEEPER_WRITE_FAILED); + } + + @Override + public Result deleteControllerPreferredCandidate(Long clusterId, Integer brokerId) { + if (ValidateUtils.isNull(clusterId)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + ZkConfigImpl zkConfig = PhysicalClusterMetadataManager.getZKConfig(clusterId); + if (ValidateUtils.isNull(zkConfig)) { + return Result.buildFrom(ResultStatus.ZOOKEEPER_CONNECT_FAILED); + } + + try { + if (!zkConfig.checkPathExists(ZkPathUtil.getControllerCandidatePath(brokerId))) { + return Result.buildSuc(); + } + zkConfig.delete(ZkPathUtil.getControllerCandidatePath(brokerId)); + return Result.buildSuc(); + } catch (Exception e) { + LOGGER.error("class=ZookeeperServiceImpl||method=deleteControllerPreferredCandidate||clusterId={}||brokerId={}||errMsg={}||", clusterId, brokerId, e.getMessage()); + } + return Result.buildFrom(ResultStatus.ZOOKEEPER_DELETE_FAILED); + } } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/utils/TopicCommands.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/utils/TopicCommands.java index 58e5d98b..6995eb97 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/utils/TopicCommands.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/utils/TopicCommands.java @@ -44,7 +44,7 @@ public class TopicCommands { ); // 生成分配策略 - scala.collection.Map> replicaAssignment = + scala.collection.Map> replicaAssignment = AdminUtils.assignReplicasToBrokers( convert2BrokerMetadataSeq(brokerIdList), partitionNum, @@ -177,7 +177,7 @@ public class TopicCommands { ) ); - Map> existingAssignJavaMap = + Map> existingAssignJavaMap = JavaConversions.asJavaMap(existingAssignScalaMap); // 新增分区的分配策略和旧的分配策略合并 Map> targetMap = new HashMap<>(); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java index f808b976..4314a101 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java @@ -5,8 +5,6 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata import com.xiaojukeji.kafka.manager.common.zookeeper.StateChangeListener; import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl; import com.xiaojukeji.kafka.manager.common.zookeeper.ZkPathUtil; -import com.xiaojukeji.kafka.manager.dao.TopicDao; -import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.ThreadPool; import org.apache.zookeeper.data.Stat; @@ -24,28 +22,17 @@ import java.util.concurrent.*; * @date 20/5/14 */ public class TopicStateListener implements StateChangeListener { - private final static Logger LOGGER = LoggerFactory.getLogger(TopicStateListener.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TopicStateListener.class); private Long clusterId; private ZkConfigImpl zkConfig; - private TopicDao topicDao; - - private AuthorityDao authorityDao; - public TopicStateListener(Long clusterId, ZkConfigImpl zkConfig) { this.clusterId = clusterId; this.zkConfig = zkConfig; } - public TopicStateListener(Long clusterId, ZkConfigImpl zkConfig, TopicDao topicDao, AuthorityDao authorityDao) { - this.clusterId = clusterId; - this.zkConfig = zkConfig; - this.topicDao = topicDao; - this.authorityDao = authorityDao; - } - @Override public void init() { try { @@ -53,7 +40,7 @@ public class TopicStateListener implements StateChangeListener { FutureTask[] taskList = new FutureTask[topicNameList.size()]; for (int i = 0; i < topicNameList.size(); i++) { String topicName = topicNameList.get(i); - taskList[i] = new FutureTask(new Callable() { + taskList[i] = new FutureTask(new Callable() { @Override public Object call() throws Exception { processTopicAdded(topicName); @@ -65,7 +52,6 @@ public class TopicStateListener implements StateChangeListener { } catch (Exception e) { LOGGER.error("init topics metadata failed, clusterId:{}.", clusterId, e); } - return; } @Override @@ -92,8 +78,6 @@ public class TopicStateListener implements StateChangeListener { private void processTopicDelete(String topicName) { LOGGER.warn("delete topic, clusterId:{} topicName:{}.", clusterId, topicName); PhysicalClusterMetadataManager.removeTopicMetadata(clusterId, topicName); - topicDao.removeTopicInCache(clusterId, topicName); - authorityDao.removeAuthorityInCache(clusterId, topicName); } private void processTopicAdded(String topicName) { @@ -122,4 +106,4 @@ public class TopicStateListener implements StateChangeListener { LOGGER.error("add topic failed, clusterId:{} topicMetadata:{}.", clusterId, topicMetadata, e); } } -} \ No newline at end of file +} diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java index 3d3f5410..64e089a6 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java @@ -22,6 +22,4 @@ public interface TopicDao { List listAll(); TopicDO getTopic(Long clusterId, String topicName, String appId); - - TopicDO removeTopicInCache(Long clusterId, String topicName); } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AppDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AppDao.java index 218c8656..7802005a 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AppDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AppDao.java @@ -16,8 +16,6 @@ public interface AppDao { */ int insert(AppDO appDO); - int insertIgnoreGatewayDB(AppDO appDO); - /** * 删除appId * @param appName App名称 @@ -60,6 +58,4 @@ public interface AppDao { * @return int */ int updateById(AppDO appDO); - - List listNewAll(); } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java index a7a8affe..655218e9 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java @@ -15,8 +15,6 @@ public interface AuthorityDao { */ int insert(AuthorityDO authorityDO); - int replaceIgnoreGatewayDB(AuthorityDO authorityDO); - /** * 获取权限 * @param clusterId 集群id @@ -38,7 +36,5 @@ public interface AuthorityDao { Map>> getAllAuthority(); - void removeAuthorityInCache(Long clusterId, String topicName); - int deleteAuthorityByTopic(Long clusterId, String topicName); } diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AppDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AppDaoImpl.java index aa08c1b4..62475b9b 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AppDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AppDaoImpl.java @@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.dao.gateway.impl; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.dao.gateway.AppDao; +import com.xiaojukeji.kafka.manager.task.Constant; import org.mybatis.spring.SqlSessionTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -21,7 +22,7 @@ public class AppDaoImpl implements AppDao { /** * APP最近的一次更新时间, 更新之后的缓存 */ - private static Long APP_CACHE_LATEST_UPDATE_TIME = 0L; + private static volatile long APP_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP; private static final Map APP_MAP = new ConcurrentHashMap<>(); @Override @@ -29,11 +30,6 @@ public class AppDaoImpl implements AppDao { return sqlSession.insert("AppDao.insert", appDO); } - @Override - public int insertIgnoreGatewayDB(AppDO appDO) { - return sqlSession.insert("AppDao.insert", appDO); - } - @Override public int deleteByName(String appName) { return sqlSession.delete("AppDao.deleteByName", appName); @@ -66,7 +62,12 @@ public class AppDaoImpl implements AppDao { } private void updateTopicCache() { - Long timestamp = System.currentTimeMillis(); + long timestamp = System.currentTimeMillis(); + + if (timestamp + 1000 <= APP_CACHE_LATEST_UPDATE_TIME) { + // 近一秒内的请求不走db + return; + } Date afterTime = new Date(APP_CACHE_LATEST_UPDATE_TIME); List doList = sqlSession.selectList("AppDao.listAfterTime", afterTime); @@ -76,19 +77,22 @@ public class AppDaoImpl implements AppDao { /** * 更新APP缓存 */ - synchronized private void updateTopicCache(List doList, Long timestamp) { + private synchronized void updateTopicCache(List doList, long timestamp) { if (doList == null || doList.isEmpty() || APP_CACHE_LATEST_UPDATE_TIME >= timestamp) { // 本次无数据更新, 或者本次更新过时 时, 忽略本次更新 return; } + if (APP_CACHE_LATEST_UPDATE_TIME == Constant.START_TIMESTAMP) { + APP_MAP.clear(); + } + for (AppDO elem: doList) { APP_MAP.put(elem.getAppId(), elem); } APP_CACHE_LATEST_UPDATE_TIME = timestamp; } - @Override - public List listNewAll() { - return sqlSession.selectList("AppDao.listNewAll"); + public static void resetCache() { + APP_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP; } } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java index 74a7cab0..1b5df873 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java @@ -1,8 +1,8 @@ package com.xiaojukeji.kafka.manager.dao.gateway.impl; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; -import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; +import com.xiaojukeji.kafka.manager.task.Constant; import org.mybatis.spring.SqlSessionTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -23,7 +23,8 @@ public class AuthorityDaoImpl implements AuthorityDao { * Authority最近的一次更新时间, 更新之后的缓存 * >> */ - private static Long AUTHORITY_CACHE_LATEST_UPDATE_TIME = 0L; + private static volatile long AUTHORITY_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP; + private static final Map>> AUTHORITY_MAP = new ConcurrentHashMap<>(); @Override @@ -31,11 +32,6 @@ public class AuthorityDaoImpl implements AuthorityDao { return sqlSession.insert("AuthorityDao.replace", authorityDO); } - @Override - public int replaceIgnoreGatewayDB(AuthorityDO authorityDO) { - return sqlSession.insert("AuthorityDao.replace", authorityDO); - } - @Override public List getAuthority(Long clusterId, String topicName, String appId) { Map params = new HashMap<>(3); @@ -62,8 +58,8 @@ public class AuthorityDaoImpl implements AuthorityDao { } List authorityDOList = new ArrayList<>(); - for (Long clusterId: doMap.keySet()) { - authorityDOList.addAll(doMap.get(clusterId).values()); + for (Map.Entry> entry: doMap.entrySet()) { + authorityDOList.addAll(entry.getValue().values()); } return authorityDOList; } @@ -87,23 +83,6 @@ public class AuthorityDaoImpl implements AuthorityDao { return AUTHORITY_MAP; } - @Override - public void removeAuthorityInCache(Long clusterId, String topicName) { - AUTHORITY_MAP.forEach((appId, map) -> { - map.forEach((id, subMap) -> { - if (id.equals(clusterId)) { - subMap.remove(topicName); - if (subMap.isEmpty()) { - map.remove(id); - } - } - }); - if (map.isEmpty()) { - AUTHORITY_MAP.remove(appId); - } - }); - } - @Override public int deleteAuthorityByTopic(Long clusterId, String topicName) { Map params = new HashMap<>(2); @@ -116,6 +95,11 @@ public class AuthorityDaoImpl implements AuthorityDao { private void updateAuthorityCache() { Long timestamp = System.currentTimeMillis(); + if (timestamp + 1000 <= AUTHORITY_CACHE_LATEST_UPDATE_TIME) { + // 近一秒内的请求不走db + return; + } + Date afterTime = new Date(AUTHORITY_CACHE_LATEST_UPDATE_TIME); List doList = sqlSession.selectList("AuthorityDao.listAfterTime", afterTime); updateAuthorityCache(doList, timestamp); @@ -124,11 +108,15 @@ public class AuthorityDaoImpl implements AuthorityDao { /** * 更新Topic缓存 */ - synchronized private void updateAuthorityCache(List doList, Long timestamp) { + private synchronized void updateAuthorityCache(List doList, Long timestamp) { if (doList == null || doList.isEmpty() || AUTHORITY_CACHE_LATEST_UPDATE_TIME >= timestamp) { // 本次无数据更新, 或者本次更新过时 时, 忽略本次更新 return; } + if (AUTHORITY_CACHE_LATEST_UPDATE_TIME == Constant.START_TIMESTAMP) { + AUTHORITY_MAP.clear(); + } + for (AuthorityDO elem: doList) { Map> doMap = AUTHORITY_MAP.getOrDefault(elem.getAppId(), new ConcurrentHashMap<>()); @@ -139,4 +127,8 @@ public class AuthorityDaoImpl implements AuthorityDao { } AUTHORITY_CACHE_LATEST_UPDATE_TIME = timestamp; } + + public static void resetCache() { + AUTHORITY_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP; + } } diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java index ba4468df..3c1ba335 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java @@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.dao.impl; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; import com.xiaojukeji.kafka.manager.dao.TopicDao; +import com.xiaojukeji.kafka.manager.task.Constant; import org.mybatis.spring.SqlSessionTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -18,7 +19,8 @@ public class TopicDaoImpl implements TopicDao { /** * Topic最近的一次更新时间, 更新之后的缓存 */ - private static Long TOPIC_CACHE_LATEST_UPDATE_TIME = 0L; + private static volatile long TOPIC_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP; + private static final Map> TOPIC_MAP = new ConcurrentHashMap<>(); @Autowired @@ -62,7 +64,7 @@ public class TopicDaoImpl implements TopicDao { @Override public List getByClusterId(Long clusterId) { updateTopicCache(); - return new ArrayList<>(TOPIC_MAP.getOrDefault(clusterId, new ConcurrentHashMap<>(0)).values()); + return new ArrayList<>(TOPIC_MAP.getOrDefault(clusterId, Collections.emptyMap()).values()); } @Override @@ -75,28 +77,28 @@ public class TopicDaoImpl implements TopicDao { updateTopicCache(); List doList = new ArrayList<>(); for (Long clusterId: TOPIC_MAP.keySet()) { - doList.addAll(TOPIC_MAP.getOrDefault(clusterId, new ConcurrentHashMap<>(0)).values()); + doList.addAll(TOPIC_MAP.getOrDefault(clusterId, Collections.emptyMap()).values()); } return doList; } @Override public TopicDO getTopic(Long clusterId, String topicName, String appId) { - Map params = new HashMap<>(2); + Map params = new HashMap<>(3); params.put("clusterId", clusterId); params.put("topicName", topicName); params.put("appId", appId); return sqlSession.selectOne("TopicDao.getTopic", params); } - @Override - public TopicDO removeTopicInCache(Long clusterId, String topicName) { - return TOPIC_MAP.getOrDefault(clusterId, new HashMap<>(0)).remove(topicName); - } - private void updateTopicCache() { Long timestamp = System.currentTimeMillis(); + if (timestamp + 1000 <= TOPIC_CACHE_LATEST_UPDATE_TIME) { + // 近一秒内的请求不走db + return; + } + Date afterTime = new Date(TOPIC_CACHE_LATEST_UPDATE_TIME); List doList = sqlSession.selectList("TopicDao.listAfterTime", afterTime); updateTopicCache(doList, timestamp); @@ -105,11 +107,15 @@ public class TopicDaoImpl implements TopicDao { /** * 更新Topic缓存 */ - synchronized private void updateTopicCache(List doList, Long timestamp) { + private synchronized void updateTopicCache(List doList, Long timestamp) { if (doList == null || doList.isEmpty() || TOPIC_CACHE_LATEST_UPDATE_TIME >= timestamp) { // 本次无数据更新, 或者本次更新过时 时, 忽略本次更新 return; } + if (TOPIC_CACHE_LATEST_UPDATE_TIME == Constant.START_TIMESTAMP) { + TOPIC_MAP.clear(); + } + for (TopicDO elem: doList) { Map doMap = TOPIC_MAP.getOrDefault(elem.getClusterId(), new ConcurrentHashMap<>()); doMap.put(elem.getTopicName(), elem); @@ -117,4 +123,8 @@ public class TopicDaoImpl implements TopicDao { } TOPIC_CACHE_LATEST_UPDATE_TIME = timestamp; } + + public static void resetCache() { + TOPIC_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP; + } } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/Constant.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/Constant.java new file mode 100644 index 00000000..3a50d7c1 --- /dev/null +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/Constant.java @@ -0,0 +1,5 @@ +package com.xiaojukeji.kafka.manager.task; + +public class Constant { + public static final long START_TIMESTAMP = 0; +} diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/DaoBackgroundTask.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/DaoBackgroundTask.java new file mode 100644 index 00000000..a750aff8 --- /dev/null +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/DaoBackgroundTask.java @@ -0,0 +1,41 @@ +package com.xiaojukeji.kafka.manager.task; + +import com.xiaojukeji.kafka.manager.common.utils.factory.DefaultThreadFactory; +import com.xiaojukeji.kafka.manager.dao.gateway.impl.AppDaoImpl; +import com.xiaojukeji.kafka.manager.dao.gateway.impl.AuthorityDaoImpl; +import com.xiaojukeji.kafka.manager.dao.impl.TopicDaoImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * 后台任务线程 + * @author zengqiao + * @date 21/02/02 + */ +@Service +public class DaoBackgroundTask { + private static final Logger LOGGER = LoggerFactory.getLogger(DaoBackgroundTask.class); + + private static final ScheduledExecutorService SYNC_CACHE_THREAD_POOL = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("syncCacheTask")); + + @PostConstruct + public void init() { + SYNC_CACHE_THREAD_POOL.scheduleAtFixedRate(() -> { + LOGGER.info("class=DaoBackgroundTask||method=init||msg=sync cache start"); + + TopicDaoImpl.resetCache(); + + AppDaoImpl.resetCache(); + + AuthorityDaoImpl.resetCache(); + + LOGGER.info("class=DaoBackgroundTask||method=init||msg=sync cache finished"); + }, 1, 10, TimeUnit.MINUTES); + } +} diff --git a/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml b/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml index 8aa91925..ac003836 100644 --- a/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml +++ b/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml @@ -8,6 +8,7 @@ + @@ -27,9 +28,9 @@ @@ -45,7 +46,8 @@ `type`=#{type}, `name`=#{name}, `value`=#{value}, - `version`=#{version} + `version`=#{version}, + `description`=#{description} WHERE id=#{id} ]]> diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java index 8b208385..7f4974ea 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java @@ -33,7 +33,7 @@ public interface AccountService { * @param username 用户名 * @return */ - ResultStatus deleteByName(String username); + ResultStatus deleteByName(String username, String operator); /** * 更新账号 diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java index 89be275b..9362d895 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java @@ -2,13 +2,16 @@ package com.xiaojukeji.kafka.manager.account.component.sso; import com.xiaojukeji.kafka.manager.account.AccountService; import com.xiaojukeji.kafka.manager.account.component.AbstractSingleSignOn; +import com.xiaojukeji.kafka.manager.common.bizenum.AccountRoleEnum; import com.xiaojukeji.kafka.manager.common.constant.LoginConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.LoginDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.AccountDO; import com.xiaojukeji.kafka.manager.common.utils.EncryptUtil; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.common.utils.ldap.LDAPAuthentication; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletRequest; @@ -23,12 +26,50 @@ public class BaseSessionSignOn extends AbstractSingleSignOn { @Autowired private AccountService accountService; + @Autowired + private LDAPAuthentication ldapAuthentication; + + //是否开启ldap验证 + @Value(value = "${ldap.enabled}") + private boolean ldapEnabled; + + //ldap自动注册的默认角色。请注意：它通常来说都是低权限角色 + @Value(value = "${ldap.auth-user-registration-role}") + private String authUserRegistrationRole; + + //ldap自动注册是否开启 + @Value(value = "${ldap.auth-user-registration}") + private boolean authUserRegistration; + @Override public Result loginAndGetLdap(HttpServletRequest request, HttpServletResponse response, LoginDTO dto) { if (ValidateUtils.isBlank(dto.getUsername()) || ValidateUtils.isNull(dto.getPassword())) { return null; } + Result accountResult = accountService.getAccountDO(dto.getUsername()); + + //modifier limin + //判断是否激活了LDAP验证。若激活并且数据库无此用户则自动注册 + if(ldapEnabled){ + //去LDAP验证账密 + if(!ldapAuthentication.authenricate(dto.getUsername(),dto.getPassword())){ + return null; + } + + if(ValidateUtils.isNull(accountResult) && authUserRegistration){ + //自动注册 + AccountDO accountDO = new AccountDO(); + accountDO.setUsername(dto.getUsername()); + accountDO.setRole(AccountRoleEnum.getUserRoleEnum(authUserRegistrationRole).getRole()); + accountDO.setPassword(EncryptUtil.md5(dto.getPassword())); + accountService.createAccount(accountDO); + } + + return Result.buildSuc(dto.getUsername()); + + } + if (ValidateUtils.isNull(accountResult) || accountResult.failed()) { return new Result<>(accountResult.getCode(), accountResult.getMessage()); } diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java index 39d773ed..e4d03c23 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java @@ -6,6 +6,8 @@ import com.xiaojukeji.kafka.manager.account.AccountService; import com.xiaojukeji.kafka.manager.account.common.EnterpriseStaff; import com.xiaojukeji.kafka.manager.account.component.AbstractEnterpriseStaffService; import com.xiaojukeji.kafka.manager.common.bizenum.AccountRoleEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.ModuleEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.OperateEnum; import com.xiaojukeji.kafka.manager.common.constant.Constant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; @@ -15,6 +17,7 @@ import com.xiaojukeji.kafka.manager.common.utils.EncryptUtil; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.dao.AccountDao; import com.xiaojukeji.kafka.manager.service.service.ConfigService; +import com.xiaojukeji.kafka.manager.service.service.OperateRecordService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -48,6 +51,9 @@ public class AccountServiceImpl implements AccountService { @Autowired private AbstractEnterpriseStaffService enterpriseStaffService; + @Autowired + private OperateRecordService operateRecordService; + /** * 用户组织信息 * @@ -82,9 +88,12 @@ public class AccountServiceImpl implements AccountService { } @Override - public ResultStatus deleteByName(String username) { + public ResultStatus deleteByName(String username, String operator) { try { if (accountDao.deleteByName(username) > 0) { + Map content = new HashMap<>(); + content.put("username", username); + operateRecordService.insert(operator, ModuleEnum.AUTHORITY, username, OperateEnum.DELETE, content); return ResultStatus.SUCCESS; } } catch (Exception e) { diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java index 0045bfe2..6a2c0bb4 100644 --- a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java @@ -18,6 +18,9 @@ public class OrderExtensionAddGatewayConfigDTO { @ApiModelProperty(value = "值") private String value; + @ApiModelProperty(value = "描述说明") + private String description; + public String getType() { return type; } @@ -42,12 +45,21 @@ public class OrderExtensionAddGatewayConfigDTO { this.value = value; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public String toString() { return "OrderExtensionAddGatewayConfigDTO{" + "type='" + type + '\'' + ", name='" + name + '\'' + ", value='" + value + '\'' + + ", description='" + description + '\'' + '}'; } diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java index f5212f8c..3f749ea7 100644 --- a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java @@ -23,6 +23,9 @@ public class OrderExtensionModifyGatewayConfigDTO { @ApiModelProperty(value = "值") private String value; + @ApiModelProperty(value = "描述说明") + private String description; + public Long getId() { return id; } @@ -55,6 +58,14 @@ public class OrderExtensionModifyGatewayConfigDTO { this.value = value; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public String toString() { return "OrderExtensionModifyGatewayConfigDTO{" + @@ -62,6 +73,7 @@ public class OrderExtensionModifyGatewayConfigDTO { ", type='" + type + '\'' + ", name='" + name + '\'' + ", value='" + value + '\'' + + ", description='" + description + '\'' + '}'; } diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/ApplyAppOrder.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/ApplyAppOrder.java index d902abed..1528ada8 100644 --- a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/ApplyAppOrder.java +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/ApplyAppOrder.java @@ -87,6 +87,6 @@ public class ApplyAppOrder extends AbstractAppOrder { appDO.setDescription(orderDO.getDescription()); appDO.generateAppIdAndPassword(orderDO.getId(), configUtils.getIdc()); appDO.setType(0); - return appService.addApp(appDO); + return appService.addApp(appDO, userName); } } diff --git a/kafka-manager-extends/kafka-manager-kcm/pom.xml b/kafka-manager-extends/kafka-manager-kcm/pom.xml index 741f0f12..7ffd00e3 100644 --- a/kafka-manager-extends/kafka-manager-kcm/pom.xml +++ b/kafka-manager-extends/kafka-manager-kcm/pom.xml @@ -68,5 +68,10 @@ spring-test ${spring-version} + + + io.minio + minio + \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/KafkaFileService.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/KafkaFileService.java index b2de3a32..babfeb15 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/KafkaFileService.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/KafkaFileService.java @@ -4,6 +4,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.KafkaFileDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.KafkaFileDO; +import org.springframework.web.multipart.MultipartFile; import java.util.List; @@ -24,7 +25,7 @@ public interface KafkaFileService { KafkaFileDO getFileByFileName(String fileName); - Result downloadKafkaConfigFile(Long fileId); + Result downloadKafkaFile(Long fileId); String getDownloadBaseUrl(); } diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/Constant.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/Constant.java new file mode 100644 index 00000000..f73c3fd6 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/Constant.java @@ -0,0 +1,18 @@ +package com.xiaojukeji.kafka.manager.kcm.common; + +public class Constant { + /** + * + */ + public static final String TASK_TITLE_PREFIX = "Logi-Kafka"; + + /** + * 并发度，顺序执行 + */ + public static final Integer AGENT_TASK_BATCH = 1; + + /** + * 失败的容忍度为0 + */ + public static final Integer AGENT_TASK_TOLERANCE = 0; +} diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/bizenum/ClusterTaskActionEnum.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/bizenum/ClusterTaskActionEnum.java index 556acab8..a51e2c68 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/bizenum/ClusterTaskActionEnum.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/bizenum/ClusterTaskActionEnum.java @@ -6,34 +6,35 @@ package com.xiaojukeji.kafka.manager.kcm.common.bizenum; * @date 20/4/26 */ public enum ClusterTaskActionEnum { - START(0, "start"), - PAUSE(1, "pause"), - IGNORE(2, "ignore"), - CANCEL(3, "cancel"), - ROLLBACK(4, "rollback"), + UNKNOWN("unknown"), + + START("start"), + PAUSE("pause"), + + IGNORE("ignore"), + CANCEL("cancel"), + + REDO("redo"), + KILL("kill"), + + ROLLBACK("rollback"), + ; - private Integer code; - private String message; + private String action; - ClusterTaskActionEnum(Integer code, String message) { - this.code = code; - this.message = message; + ClusterTaskActionEnum(String action) { + this.action = action; } - public Integer getCode() { - return code; - } - - public String getMessage() { - return message; + public String getAction() { + return action; } @Override public String toString() { - return "TaskActionEnum{" + - "code=" + code + - ", message='" + message + '\'' + + return "ClusterTaskActionEnum{" + + "action='" + action + '\'' + '}'; } } diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/ClusterTaskLog.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/ClusterTaskLog.java new file mode 100644 index 00000000..ff89fa99 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/ClusterTaskLog.java @@ -0,0 +1,24 @@ +package com.xiaojukeji.kafka.manager.kcm.common.entry.ao; + +public class ClusterTaskLog { + private String stdout; + + public ClusterTaskLog(String stdout) { + this.stdout = stdout; + } + + public String getStdout() { + return stdout; + } + + public void setStdout(String stdout) { + this.stdout = stdout; + } + + @Override + public String toString() { + return "AgentOperationTaskLog{" + + "stdout='" + stdout + '\'' + + '}'; + } +} diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/CreationTaskData.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/CreationTaskData.java index bc025d5c..8c2cd1ec 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/CreationTaskData.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/common/entry/ao/CreationTaskData.java @@ -1,5 +1,7 @@ package com.xiaojukeji.kafka.manager.kcm.common.entry.ao; +import com.xiaojukeji.kafka.manager.common.entity.Result; + import java.util.List; /** @@ -119,7 +121,7 @@ public class CreationTaskData { @Override public String toString() { - return "CreationTaskDTO{" + + return "CreationTaskData{" + "uuid='" + uuid + '\'' + ", clusterId=" + clusterId + ", hostList=" + hostList + diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/AbstractAgent.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/AbstractAgent.java index 88872868..70ce5902 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/AbstractAgent.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/AbstractAgent.java @@ -1,9 +1,18 @@ package com.xiaojukeji.kafka.manager.kcm.component.agent; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskActionEnum; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskStateEnum; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskSubStateEnum; +import com.xiaojukeji.kafka.manager.kcm.common.entry.ao.ClusterTaskLog; import com.xiaojukeji.kafka.manager.kcm.common.entry.ao.CreationTaskData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.Map; @@ -13,33 +22,79 @@ import java.util.Map; * @date 20/4/26 */ public abstract class AbstractAgent { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAgent.class); + /** * 创建任务 + * @param creationTaskData 创建任务参数 + * @return 任务ID */ - public abstract Long createTask(CreationTaskData dto); + public abstract Result createTask(CreationTaskData creationTaskData); /** - * 任务动作 + * 执行任务 + * @param taskId 任务ID + * @param actionEnum 执行动作 + * @return true:触发成功, false:触发失败 */ - public abstract Boolean actionTask(Long taskId, String action); + public abstract boolean actionTask(Long taskId, ClusterTaskActionEnum actionEnum); /** - * 任务动作 + * 执行任务 + * @param taskId 任务ID + * @param actionEnum 执行动作 + * @param hostname 具体主机 + * @return true:触发成功, false:触发失败 */ - public abstract Boolean actionHostTask(Long taskId, String action, String hostname); + public abstract boolean actionHostTask(Long taskId, ClusterTaskActionEnum actionEnum, String hostname); /** - * 获取任务状态 + * 获取任务运行的状态[阻塞, 执行中, 完成等] + * @param taskId 任务ID + * @return 任务状态 */ - public abstract ClusterTaskStateEnum getTaskState(Long agentTaskId); + public abstract Result getTaskExecuteState(Long taskId); /** * 获取任务结果 + * @param taskId 任务ID + * @return 任务结果 */ - public abstract Map getTaskResult(Long taskId); + public abstract Result> getTaskResult(Long taskId); /** - * 获取任务日志 + * 获取任务执行日志 + * @param taskId 任务ID + * @param hostname 具体主机 + * @return 机器运行日志 */ - public abstract String getTaskLog(Long agentTaskId, String hostname); + public abstract Result getTaskLog(Long taskId, String hostname); + + protected static String readScriptInJarFile(String fileName) { + InputStream inputStream = AbstractAgent.class.getClassLoader().getResourceAsStream(fileName); + if (inputStream == null) { + LOGGER.error("class=AbstractAgent||method=readScriptInJarFile||fileName={}||msg=read script failed", fileName); + return ""; + } + + try { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + String line = null; + + StringBuilder sb = new StringBuilder(); + while ((line = bufferedReader.readLine()) != null) { + sb.append(line).append("\n"); + } + return sb.toString(); + } catch (Exception e) { + LOGGER.error("class=AbstractAgent||method=readScriptInJarFile||fileName={}||errMsg={}||msg=read script failed", fileName, e.getMessage()); + } finally { + try { + inputStream.close(); + } catch (IOException e) { + LOGGER.error("class=AbstractAgent||method=readScriptInJarFile||fileName={}||errMsg={}||msg=close reading script failed", fileName, e.getMessage()); + } + } + return ""; + } } \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/N9e.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/N9e.java index f1f4b586..6e3fa677 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/N9e.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/N9e.java @@ -1,8 +1,11 @@ package com.xiaojukeji.kafka.manager.kcm.component.agent.n9e; -import com.alibaba.fastjson.JSON; import com.xiaojukeji.kafka.manager.common.bizenum.KafkaFileEnum; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.kcm.common.Constant; +import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskActionEnum; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskTypeEnum; +import com.xiaojukeji.kafka.manager.kcm.common.entry.ao.ClusterTaskLog; import com.xiaojukeji.kafka.manager.kcm.common.entry.ao.CreationTaskData; import com.xiaojukeji.kafka.manager.common.utils.HttpUtils; import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; @@ -11,20 +14,17 @@ import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskStateEnum; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskSubStateEnum; import com.xiaojukeji.kafka.manager.kcm.component.agent.AbstractAgent; +import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eCreationTask; import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eResult; -import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eTaskResultDTO; -import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eTaskStatusEnum; -import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eTaskStdoutDTO; +import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eTaskResult; +import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.N9eTaskStdoutLog; +import com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.bizenum.N9eTaskStatusEnum; import org.springframework.beans.factory.annotation.Value; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -54,16 +54,6 @@ public class N9e extends AbstractAgent { private String script; - /** - * 并发度，顺序执行 - */ - private static final Integer BATCH = 1; - - /** - * 失败的容忍度为0 - */ - private static final Integer TOLERANCE = 0; - private static final String CREATE_TASK_URI = "/api/job-ce/tasks"; private static final String ACTION_TASK_URI = "/api/job-ce/task/{taskId}/action"; @@ -82,143 +72,134 @@ public class N9e extends AbstractAgent { } @Override - public Long createTask(CreationTaskData creationTaskData) { - Map param = buildCreateTaskParam(creationTaskData); + public Result createTask(CreationTaskData creationTaskData) { + String content = JsonUtils.toJSONString(buildCreateTaskParam(creationTaskData)); String response = null; try { - response = HttpUtils.postForString( - baseUrl + CREATE_TASK_URI, - JsonUtils.toJSONString(param), - buildHeader() - ); - N9eResult zr = JSON.parseObject(response, N9eResult.class); - if (!ValidateUtils.isBlank(zr.getErr())) { - LOGGER.warn("class=N9e||method=createTask||param={}||errMsg={}||msg=call create task fail", JsonUtils.toJSONString(param),zr.getErr()); - return null; + response = HttpUtils.postForString(baseUrl + CREATE_TASK_URI, content, buildHeader()); + N9eResult nr = JsonUtils.stringToObj(response, N9eResult.class); + if (!ValidateUtils.isBlank(nr.getErr())) { + LOGGER.error("class=N9e||method=createTask||param={}||response={}||msg=call create task failed", content, response); + return Result.buildFailure(nr.getErr()); } - return Long.valueOf(zr.getDat().toString()); + return Result.buildSuc(Long.valueOf(nr.getDat().toString())); } catch (Exception e) { - LOGGER.error("create task failed, req:{}.", creationTaskData, e); + LOGGER.error("class=N9e||method=createTask||param={}||response={}||errMsg={}||msg=call create task failed", content, response, e.getMessage()); } - return null; + return Result.buildFailure("create n9e task failed"); } @Override - public Boolean actionTask(Long taskId, String action) { + public boolean actionTask(Long taskId, ClusterTaskActionEnum actionEnum) { Map param = new HashMap<>(1); - param.put("action", action); + param.put("action", actionEnum.getAction()); String response = null; try { - response = HttpUtils.putForString( - baseUrl + ACTION_TASK_URI.replace("{taskId}", taskId.toString()), - JSON.toJSONString(param), - buildHeader() - ); - N9eResult zr = JSON.parseObject(response, N9eResult.class); - if (ValidateUtils.isBlank(zr.getErr())) { + response = HttpUtils.putForString(baseUrl + ACTION_TASK_URI.replace("{taskId}", String.valueOf(taskId)), JsonUtils.toJSONString(param), buildHeader()); + N9eResult nr = JsonUtils.stringToObj(response, N9eResult.class); + if (ValidateUtils.isBlank(nr.getErr())) { return true; } - LOGGER.warn("class=N9e||method=actionTask||param={}||errMsg={}||msg=call action task fail", JSON.toJSONString(param),zr.getErr()); + + LOGGER.error("class=N9e||method=actionTask||param={}||response={}||msg=call action task fail", JsonUtils.toJSONString(param), response); return false; } catch (Exception e) { - LOGGER.error("action task failed, taskId:{}, action:{}.", taskId, action, e); + LOGGER.error("class=N9e||method=actionTask||param={}||response={}||errMsg={}||msg=call action task fail", JsonUtils.toJSONString(param), response, e.getMessage()); } return false; } @Override - public Boolean actionHostTask(Long taskId, String action, String hostname) { - Map param = new HashMap<>(2); - param.put("action", action); - param.put("hostname", hostname); + public boolean actionHostTask(Long taskId, ClusterTaskActionEnum actionEnum, String hostname) { + Map params = new HashMap<>(2); + params.put("action", actionEnum.getAction()); + params.put("hostname", hostname); String response = null; try { - response = HttpUtils.putForString( - baseUrl + ACTION_HOST_TASK_URI.replace("{taskId}", taskId.toString()), - JSON.toJSONString(param), - buildHeader() - ); - N9eResult zr = JSON.parseObject(response, N9eResult.class); - if (ValidateUtils.isBlank(zr.getErr())) { + response = HttpUtils.putForString(baseUrl + ACTION_HOST_TASK_URI.replace("{taskId}", String.valueOf(taskId)), JsonUtils.toJSONString(params), buildHeader()); + N9eResult nr = JsonUtils.stringToObj(response, N9eResult.class); + if (ValidateUtils.isBlank(nr.getErr())) { return true; } - LOGGER.warn("class=N9e||method=actionHostTask||param={}||errMsg={}||msg=call action host task fail", JSON.toJSONString(param),zr.getErr()); + + LOGGER.error("class=N9e||method=actionHostTask||params={}||response={}||msg=call action host task fail", JsonUtils.toJSONString(params), response); return false; } catch (Exception e) { - LOGGER.error("action task failed, taskId:{} action:{} hostname:{}.", taskId, action, hostname, e); + LOGGER.error("class=N9e||method=actionHostTask||params={}||response={}||errMsg={}||msg=call action host task fail", JsonUtils.toJSONString(params), response, e.getMessage()); } return false; } @Override - public ClusterTaskStateEnum getTaskState(Long agentTaskId) { + public Result getTaskExecuteState(Long taskId) { String response = null; try { // 获取任务的state - response = HttpUtils.get( - baseUrl + TASK_STATE_URI.replace("{taskId}", agentTaskId.toString()), null - ); - N9eResult n9eResult = JSON.parseObject(response, N9eResult.class); - if (!ValidateUtils.isBlank(n9eResult.getErr())) { - LOGGER.error("get response result failed, agentTaskId:{} response:{}.", agentTaskId, response); - return null; + response = HttpUtils.get(baseUrl + TASK_STATE_URI.replace("{taskId}", String.valueOf(taskId)), null); + N9eResult nr = JsonUtils.stringToObj(response, N9eResult.class); + if (!ValidateUtils.isBlank(nr.getErr())) { + return Result.buildFailure(nr.getErr()); } - String state = JSON.parseObject(JSON.toJSONString(n9eResult.getDat()), String.class); + + String state = JsonUtils.stringToObj(JsonUtils.toJSONString(nr.getDat()), String.class); + N9eTaskStatusEnum n9eTaskStatusEnum = N9eTaskStatusEnum.getByMessage(state); if (ValidateUtils.isNull(n9eTaskStatusEnum)) { - LOGGER.error("get task status failed, agentTaskId:{} state:{}.", agentTaskId, state); - return null; + LOGGER.error("class=N9e||method=getTaskExecuteState||taskId={}||response={}||msg=get task state failed", taskId, response); + return Result.buildFailure("unknown state, state:" + state); } - return n9eTaskStatusEnum.getStatus(); + return Result.buildSuc(n9eTaskStatusEnum.getStatus()); } catch (Exception e) { - LOGGER.error("get task status failed, agentTaskId:{} response:{}.", agentTaskId, response, e); + LOGGER.error("class=N9e||method=getTaskExecuteState||taskId={}||response={}||errMsg={}||msg=get task state failed", taskId, response, e.getMessage()); } - return null; + return Result.buildFailure("get task state failed"); } @Override - public Map getTaskResult(Long agentTaskId) { + public Result> getTaskResult(Long taskId) { String response = null; try { // 获取子任务的state - response = HttpUtils.get(baseUrl + TASK_SUB_STATE_URI.replace("{taskId}", agentTaskId.toString()), null); - N9eResult n9eResult = JSON.parseObject(response, N9eResult.class); + response = HttpUtils.get(baseUrl + TASK_SUB_STATE_URI.replace("{taskId}", String.valueOf(taskId)), null); + N9eResult nr = JsonUtils.stringToObj(response, N9eResult.class); + if (!ValidateUtils.isBlank(nr.getErr())) { + LOGGER.error("class=N9e||method=getTaskResult||taskId={}||response={}||msg=get task result failed", taskId, response); + return Result.buildFailure(nr.getErr()); + } - N9eTaskResultDTO n9eTaskResultDTO = - JSON.parseObject(JSON.toJSONString(n9eResult.getDat()), N9eTaskResultDTO.class); - return n9eTaskResultDTO.convert2HostnameStatusMap(); + return Result.buildSuc(JsonUtils.stringToObj(JsonUtils.toJSONString(nr.getDat()), N9eTaskResult.class).convert2HostnameStatusMap()); } catch (Exception e) { - LOGGER.error("get task result failed, agentTaskId:{} response:{}.", agentTaskId, response, e); + LOGGER.error("class=N9e||method=getTaskResult||taskId={}||response={}||errMsg={}||msg=get task result failed", taskId, response, e.getMessage()); } - return null; + return Result.buildFailure("get task result failed"); } @Override - public String getTaskLog(Long agentTaskId, String hostname) { + public Result getTaskLog(Long taskId, String hostname) { + Map params = new HashMap<>(1); + params.put("hostname", hostname); + String response = null; try { - Map params = new HashMap<>(1); - params.put("hostname", hostname); + response = HttpUtils.get(baseUrl + TASK_STD_LOG_URI.replace("{taskId}", String.valueOf(taskId)), params); + N9eResult nr = JsonUtils.stringToObj(response, N9eResult.class); + if (!ValidateUtils.isBlank(nr.getErr())) { + LOGGER.error("class=N9e||method=getTaskLog||taskId={}||response={}||msg=get task log failed", taskId, response); + return Result.buildFailure(nr.getErr()); + } - response = HttpUtils.get(baseUrl + TASK_STD_LOG_URI.replace("{taskId}", agentTaskId.toString()), params); - N9eResult n9eResult = JSON.parseObject(response, N9eResult.class); - if (!ValidateUtils.isBlank(n9eResult.getErr())) { - LOGGER.error("get task log failed, agentTaskId:{} response:{}.", agentTaskId, response); - return null; - } - List dtoList = - JSON.parseArray(JSON.toJSONString(n9eResult.getDat()), N9eTaskStdoutDTO.class); + List dtoList = JsonUtils.stringToArrObj(JsonUtils.toJSONString(nr.getDat()), N9eTaskStdoutLog.class); if (ValidateUtils.isEmptyList(dtoList)) { - return ""; + return Result.buildSuc(new ClusterTaskLog("")); } - return dtoList.get(0).getStdout(); + return Result.buildSuc(new ClusterTaskLog(dtoList.get(0).getStdout())); } catch (Exception e) { - LOGGER.error("get task log failed, agentTaskId:{}.", agentTaskId, e); + LOGGER.error("class=N9e||method=getTaskLog||taskId={}||response={}||errMsg={}||msg=get task log failed", taskId, response, e.getMessage()); } - return null; + return Result.buildFailure("get task log failed"); } private Map buildHeader() { @@ -228,7 +209,7 @@ public class N9e extends AbstractAgent { return headers; } - private Map buildCreateTaskParam(CreationTaskData creationTaskData) { + private N9eCreationTask buildCreateTaskParam(CreationTaskData creationTaskData) { StringBuilder sb = new StringBuilder(); sb.append(creationTaskData.getUuid()).append(",,"); sb.append(creationTaskData.getClusterId()).append(",,"); @@ -240,46 +221,17 @@ public class N9e extends AbstractAgent { sb.append(creationTaskData.getServerPropertiesMd5()).append(",,"); sb.append(creationTaskData.getServerPropertiesUrl()); - Map params = new HashMap<>(10); - params.put("title", String.format("集群ID=%d-升级部署", creationTaskData.getClusterId())); - params.put("batch", BATCH); - params.put("tolerance", TOLERANCE); - params.put("timeout", timeout); - params.put("pause", ListUtils.strList2String(creationTaskData.getPauseList())); - params.put("script", this.script); - params.put("args", sb.toString()); - params.put("account", account); - params.put("action", "pause"); - params.put("hosts", creationTaskData.getHostList()); - return params; - } - - private static String readScriptInJarFile(String fileName) { - InputStream inputStream = N9e.class.getClassLoader().getResourceAsStream(fileName); - if (inputStream == null) { - LOGGER.error("read kcm script failed, filename:{}", fileName); - return ""; - } - - try { - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); - String line = null; - StringBuilder stringBuilder = new StringBuilder(""); - - while ((line = bufferedReader.readLine()) != null) { - stringBuilder.append(line); - stringBuilder.append("\n"); - } - return stringBuilder.toString(); - } catch (IOException e) { - LOGGER.error("read kcm script failed, filename:{}", fileName, e); - return ""; - } finally { - try { - inputStream.close(); - } catch (IOException e) { - LOGGER.error("close reading kcm script failed, filename:{}", fileName, e); - } - } + N9eCreationTask n9eCreationTask = new N9eCreationTask(); + n9eCreationTask.setTitle(Constant.TASK_TITLE_PREFIX + "-集群ID:" + creationTaskData.getClusterId()); + n9eCreationTask.setBatch(Constant.AGENT_TASK_BATCH); + n9eCreationTask.setTolerance(Constant.AGENT_TASK_TOLERANCE); + n9eCreationTask.setTimeout(this.timeout); + n9eCreationTask.setPause(ListUtils.strList2String(creationTaskData.getPauseList())); + n9eCreationTask.setScript(this.script); + n9eCreationTask.setArgs(sb.toString()); + n9eCreationTask.setAccount(this.account); + n9eCreationTask.setAction(ClusterTaskActionEnum.PAUSE.getAction()); + n9eCreationTask.setHosts(creationTaskData.getHostList()); + return n9eCreationTask; } } \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eCreationTask.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eCreationTask.java new file mode 100644 index 00000000..6ca4c85c --- /dev/null +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eCreationTask.java @@ -0,0 +1,151 @@ +package com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry; + +import java.util.List; + +public class N9eCreationTask { + /** + * 任务标题 + */ + private String title; + + /** + * 并发度, =2则表示两台并发执行 + */ + private Integer batch; + + /** + * 错误容忍度, 达到容忍度之上时, 任务会被暂停并不可以继续执行 + */ + private Integer tolerance; + + /** + * 单台任务的超时时间(秒) + */ + private Integer timeout; + + /** + * 暂停点, 格式: host1,host2,host3 + */ + private String pause; + + /** + * 任务执行对应的脚本 + */ + private String script; + + /** + * 任务参数 + */ + private String args; + + /** + * 使用的账号 + */ + private String account; + + /** + * 动作 + */ + private String action; + + /** + * 操作的主机列表 + */ + private List hosts; + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public Integer getBatch() { + return batch; + } + + public void setBatch(Integer batch) { + this.batch = batch; + } + + public Integer getTolerance() { + return tolerance; + } + + public void setTolerance(Integer tolerance) { + this.tolerance = tolerance; + } + + public Integer getTimeout() { + return timeout; + } + + public void setTimeout(Integer timeout) { + this.timeout = timeout; + } + + public String getPause() { + return pause; + } + + public void setPause(String pause) { + this.pause = pause; + } + + public String getScript() { + return script; + } + + public void setScript(String script) { + this.script = script; + } + + public String getArgs() { + return args; + } + + public void setArgs(String args) { + this.args = args; + } + + public String getAccount() { + return account; + } + + public void setAccount(String account) { + this.account = account; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public List getHosts() { + return hosts; + } + + public void setHosts(List hosts) { + this.hosts = hosts; + } + + @Override + public String toString() { + return "N9eCreationTask{" + + "title='" + title + '\'' + + ", batch=" + batch + + ", tolerance=" + tolerance + + ", timeout=" + timeout + + ", pause='" + pause + '\'' + + ", script='" + script + '\'' + + ", args='" + args + '\'' + + ", account='" + account + '\'' + + ", action='" + action + '\'' + + ", hosts=" + hosts + + '}'; + } +} diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskResultDTO.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskResult.java similarity index 99% rename from kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskResultDTO.java rename to kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskResult.java index b787f016..e0e67b0e 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskResultDTO.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskResult.java @@ -12,7 +12,7 @@ import java.util.Map; * @author zengqiao * @date 20/9/7 */ -public class N9eTaskResultDTO { +public class N9eTaskResult { private List waiting; private List running; diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskStdoutLog.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskStdoutLog.java new file mode 100644 index 00000000..622aaa3e --- /dev/null +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/N9eTaskStdoutLog.java @@ -0,0 +1,35 @@ +package com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry; + +/** + * @author zengqiao + * @date 20/9/7 + */ +public class N9eTaskStdoutLog { + private String host; + + private String stdout; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getStdout() { + return stdout; + } + + public void setStdout(String stdout) { + this.stdout = stdout; + } + + @Override + public String toString() { + return "N9eTaskStdoutDTO{" + + "host='" + host + '\'' + + ", stdout='" + stdout + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/bizenum/N9eTaskStatusEnum.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/bizenum/N9eTaskStatusEnum.java new file mode 100644 index 00000000..4453e703 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/entry/bizenum/N9eTaskStatusEnum.java @@ -0,0 +1,59 @@ +package com.xiaojukeji.kafka.manager.kcm.component.agent.n9e.entry.bizenum; + +import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskStateEnum; + +/** + * @author zengqiao + * @date 20/9/3 + */ +public enum N9eTaskStatusEnum { + DONE(0, "done", ClusterTaskStateEnum.FINISHED), + PAUSE(1, "pause", ClusterTaskStateEnum.BLOCKED), + START(2, "start", ClusterTaskStateEnum.RUNNING), + ; + + private Integer code; + + private String message; + + private ClusterTaskStateEnum status; + + N9eTaskStatusEnum(Integer code, String message, ClusterTaskStateEnum status) { + this.code = code; + this.message = message; + this.status = status; + } + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public ClusterTaskStateEnum getStatus() { + return status; + } + + public void setStatus(ClusterTaskStateEnum status) { + this.status = status; + } + + public static N9eTaskStatusEnum getByMessage(String message) { + for (N9eTaskStatusEnum elem: N9eTaskStatusEnum.values()) { + if (elem.message.equals(message)) { + return elem; + } + } + return null; + } +} \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/AbstractStorageService.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/AbstractStorageService.java index 90192b0b..34c209ac 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/AbstractStorageService.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/AbstractStorageService.java @@ -10,13 +10,20 @@ import org.springframework.web.multipart.MultipartFile; public abstract class AbstractStorageService { /** * 上传 + * @param fileName 文件名 + * @param fileMd5 文件md5 + * @param uploadFile 文件 + * @return 上传结果 */ public abstract boolean upload(String fileName, String fileMd5, MultipartFile uploadFile); /** - * 下载 + * 下载文件 + * @param fileName 文件名 + * @param fileMd5 文件md5 + * @return 文件 */ - public abstract Result download(String fileName, String fileMd5); + public abstract Result download(String fileName, String fileMd5); /** * 下载base地址 diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/local/Local.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/local/Local.java deleted file mode 100644 index 992c09e4..00000000 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/local/Local.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.xiaojukeji.kafka.manager.kcm.component.storage.local; - -import com.xiaojukeji.kafka.manager.common.entity.Result; -import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import com.xiaojukeji.kafka.manager.kcm.component.storage.AbstractStorageService; -import org.springframework.web.multipart.MultipartFile; - -/** - * @author zengqiao - * @date 20/9/17 - */ -@Service("storageService") -public class Local extends AbstractStorageService { - @Value("${kcm.storage.base-url}") - private String baseUrl; - - @Override - public boolean upload(String fileName, String fileMd5, MultipartFile uploadFile) { - return false; - } - - @Override - public Result download(String fileName, String fileMd5) { - return Result.buildFrom(ResultStatus.STORAGE_DOWNLOAD_FILE_FAILED); - } - - @Override - public String getDownloadBaseUrl() { - return baseUrl; - } -} \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/s3/S3Service.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/s3/S3Service.java new file mode 100644 index 00000000..9519efd2 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/storage/s3/S3Service.java @@ -0,0 +1,128 @@ +package com.xiaojukeji.kafka.manager.kcm.component.storage.s3; + +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.kcm.component.storage.AbstractStorageService; +import io.minio.*; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; + + +@Service("storageService") +public class S3Service extends AbstractStorageService { + private final static Logger LOGGER = LoggerFactory.getLogger(S3Service.class); + + @Value("${kcm.s3.endpoint:}") + private String endpoint; + + @Value("${kcm.s3.access-key:}") + private String accessKey; + + @Value("${kcm.s3.secret-key:}") + private String secretKey; + + @Value("${kcm.s3.bucket:}") + private String bucket; + + private MinioClient minioClient; + + @PostConstruct + public void init() { + try { + if (ValidateUtils.anyBlank(this.endpoint, this.accessKey, this.secretKey, this.bucket)) { + // without config s3 + return; + } + minioClient = new MinioClient(endpoint, accessKey, secretKey); + } catch (Exception e) { + LOGGER.error("class=S3Service||method=init||fields={}||errMsg={}", this.toString(), e.getMessage()); + } + } + + @Override + public boolean upload(String fileName, String fileMd5, MultipartFile uploadFile) { + InputStream inputStream = null; + try { + if (!createBucketIfNotExist()) { + return false; + } + + inputStream = uploadFile.getInputStream(); + minioClient.putObject(PutObjectArgs.builder() + .bucket(this.bucket) + .object(fileName) + .stream(inputStream, inputStream.available(), -1) + .build() + ); + return true; + } catch (Exception e) { + LOGGER.error("class=S3Service||method=upload||fileName={}||errMsg={}||msg=upload failed", fileName, e.getMessage()); + } finally { + if (inputStream != null) { + try { + inputStream.close(); + } catch (IOException e) { + ; // ignore + } + } + } + return false; + } + + @Override + public Result download(String fileName, String fileMd5) { + try { + final ObjectStat stat = minioClient.statObject(this.bucket, fileName); + + InputStream is = minioClient.getObject(this.bucket, fileName); + + return Result.buildSuc(new MockMultipartFile(fileName, fileName, stat.contentType(), is)); + } catch (Exception e) { + LOGGER.error("class=S3Service||method=download||fileName={}||errMsg={}||msg=download failed", fileName, e.getMessage()); + } + return Result.buildFrom(ResultStatus.STORAGE_DOWNLOAD_FILE_FAILED); + } + + @Override + public String getDownloadBaseUrl() { + if (this.endpoint.startsWith("http://")) { + return this.endpoint + "/" + this.bucket; + } + return "http://" + this.endpoint + "/" + this.bucket; + } + + private boolean createBucketIfNotExist() { + try { + boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(this.bucket).build()); + if (!found) { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(this.bucket).build()); + } + + LOGGER.info("class=S3Service||method=createBucketIfNotExist||bucket={}||msg=check and create bucket success", this.bucket); + return true; + } catch (Exception e) { + LOGGER.error("class=S3Service||method=createBucketIfNotExist||bucket={}||errMsg={}||msg=create bucket failed", this.bucket, e.getMessage()); + } + return false; + } + + @Override + public String toString() { + return "S3Service{" + + "endpoint='" + endpoint + '\'' + + ", accessKey='" + accessKey + '\'' + + ", secretKey='" + secretKey + '\'' + + ", bucket='" + bucket + '\'' + + '}'; + } +} diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/ClusterTaskServiceImpl.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/ClusterTaskServiceImpl.java index a190350a..b3ef959a 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/ClusterTaskServiceImpl.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/ClusterTaskServiceImpl.java @@ -6,6 +6,7 @@ import com.xiaojukeji.kafka.manager.kcm.ClusterTaskService; import com.xiaojukeji.kafka.manager.kcm.common.Converters; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskActionEnum; import com.xiaojukeji.kafka.manager.kcm.common.entry.ClusterTaskConstant; +import com.xiaojukeji.kafka.manager.kcm.common.entry.ao.ClusterTaskLog; import com.xiaojukeji.kafka.manager.kcm.common.entry.ao.ClusterTaskSubStatus; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskStateEnum; import com.xiaojukeji.kafka.manager.kcm.common.bizenum.ClusterTaskSubStateEnum; @@ -34,7 +35,7 @@ import java.util.*; */ @Service("clusterTaskService") public class ClusterTaskServiceImpl implements ClusterTaskService { - private final static Logger LOGGER = LoggerFactory.getLogger(ClusterTaskServiceImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ClusterTaskServiceImpl.class); @Autowired private AbstractAgent abstractAgent; @@ -63,13 +64,13 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { } // 创建任务 - Long agentTaskId = abstractAgent.createTask(dtoResult.getData()); - if (ValidateUtils.isNull(agentTaskId)) { + Result createResult = abstractAgent.createTask(dtoResult.getData()); + if (ValidateUtils.isNull(createResult) || createResult.failed()) { return Result.buildFrom(ResultStatus.CALL_CLUSTER_TASK_AGENT_FAILED); } try { - if (clusterTaskDao.insert(Converters.convert2ClusterTaskDO(agentTaskId, dtoResult.getData(), operator)) > 0) { + if (clusterTaskDao.insert(Converters.convert2ClusterTaskDO(createResult.getData(), dtoResult.getData(), operator)) > 0) { return Result.buildFrom(ResultStatus.SUCCESS); } } catch (Exception e) { @@ -87,45 +88,44 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { Long agentTaskId = getActiveAgentTaskId(clusterTaskDO); Boolean rollback = inRollback(clusterTaskDO); - ClusterTaskStateEnum stateEnum = abstractAgent.getTaskState(agentTaskId); - if (ClusterTaskActionEnum.START.getMessage().equals(action) - && ClusterTaskStateEnum.BLOCKED.equals(stateEnum)) { + Result stateEnumResult = abstractAgent.getTaskExecuteState(agentTaskId); + if (ValidateUtils.isNull(stateEnumResult) || stateEnumResult.failed()) { + return ResultStatus.CALL_CLUSTER_TASK_AGENT_FAILED; + } + + if (ClusterTaskActionEnum.START.getAction().equals(action) && ClusterTaskStateEnum.BLOCKED.equals(stateEnumResult.getData())) { // 暂停状态, 可以执行开始 - return actionTaskExceptRollbackAction(agentTaskId, action, ""); + return actionTaskExceptRollbackAction(agentTaskId, ClusterTaskActionEnum.START, ""); } - if (ClusterTaskActionEnum.PAUSE.getMessage().equals(action) - && ClusterTaskStateEnum.RUNNING.equals(stateEnum)) { + if (ClusterTaskActionEnum.PAUSE.getAction().equals(action) && ClusterTaskStateEnum.RUNNING.equals(stateEnumResult.getData())) { // 运行状态, 可以执行暂停 - return actionTaskExceptRollbackAction(agentTaskId, action, ""); + return actionTaskExceptRollbackAction(agentTaskId, ClusterTaskActionEnum.PAUSE, ""); } - if (ClusterTaskActionEnum.IGNORE.getMessage().equals(action) - || ClusterTaskActionEnum.CANCEL.getMessage().equals(action)) { + if (ClusterTaskActionEnum.IGNORE.getAction().equals(action)) { // 忽略 & 取消随时都可以操作 - return actionTaskExceptRollbackAction(agentTaskId, action, hostname); + return actionTaskExceptRollbackAction(agentTaskId, ClusterTaskActionEnum.IGNORE, hostname); } - if ((!ClusterTaskStateEnum.FINISHED.equals(stateEnum) || !rollback) - && ClusterTaskActionEnum.ROLLBACK.getMessage().equals(action)) { + if (ClusterTaskActionEnum.CANCEL.getAction().equals(action)) { + // 忽略 & 取消随时都可以操作 + return actionTaskExceptRollbackAction(agentTaskId, ClusterTaskActionEnum.CANCEL, hostname); + } + if ((!ClusterTaskStateEnum.FINISHED.equals(stateEnumResult.getData()) || !rollback) + && ClusterTaskActionEnum.ROLLBACK.getAction().equals(action)) { // 暂未操作完时可以回滚, 回滚所有操作过的机器到上一个版本 return actionTaskRollback(clusterTaskDO); } return ResultStatus.OPERATION_FAILED; } - private ResultStatus actionTaskExceptRollbackAction(Long agentId, String action, String hostname) { + private ResultStatus actionTaskExceptRollbackAction(Long agentId, ClusterTaskActionEnum actionEnum, String hostname) { if (!ValidateUtils.isBlank(hostname)) { - return actionHostTaskExceptRollbackAction(agentId, action, hostname); + return actionHostTaskExceptRollbackAction(agentId, actionEnum, hostname); } - if (abstractAgent.actionTask(agentId, action)) { - return ResultStatus.SUCCESS; - } - return ResultStatus.OPERATION_FAILED; + return abstractAgent.actionTask(agentId, actionEnum)? ResultStatus.SUCCESS: ResultStatus.OPERATION_FAILED; } - private ResultStatus actionHostTaskExceptRollbackAction(Long agentId, String action, String hostname) { - if (abstractAgent.actionHostTask(agentId, action, hostname)) { - return ResultStatus.SUCCESS; - } - return ResultStatus.OPERATION_FAILED; + private ResultStatus actionHostTaskExceptRollbackAction(Long agentId, ClusterTaskActionEnum actionEnum, String hostname) { + return abstractAgent.actionHostTask(agentId, actionEnum, hostname)? ResultStatus.SUCCESS: ResultStatus.OPERATION_FAILED; } private ResultStatus actionTaskRollback(ClusterTaskDO clusterTaskDO) { @@ -133,9 +133,9 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { return ResultStatus.OPERATION_FORBIDDEN; } - Map subStatusEnumMap = + Result> subStatusEnumMapResult = abstractAgent.getTaskResult(clusterTaskDO.getAgentTaskId()); - if (ValidateUtils.isNull(subStatusEnumMap)) { + if (ValidateUtils.isNull(subStatusEnumMapResult) || subStatusEnumMapResult.failed()) { return ResultStatus.CALL_CLUSTER_TASK_AGENT_FAILED; } @@ -143,7 +143,7 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { List rollbackHostList = new ArrayList<>(); List rollbackPauseHostList = new ArrayList<>(); for (String host: ListUtils.string2StrList(clusterTaskDO.getHostList())) { - ClusterTaskSubStateEnum subStateEnum = subStatusEnumMap.get(host); + ClusterTaskSubStateEnum subStateEnum = subStatusEnumMapResult.getData().get(host); if (ValidateUtils.isNull(subStateEnum)) { // 机器对应的任务查询失败 return ResultStatus.OPERATION_FAILED; @@ -166,17 +166,17 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { clusterTaskDO.setRollbackPauseHostList(ListUtils.strList2String(rollbackPauseHostList)); // 创建任务 - Long agentTaskId = abstractAgent.createTask(Converters.convert2CreationTaskData(clusterTaskDO)); - if (ValidateUtils.isNull(agentTaskId)) { + Result createResult = abstractAgent.createTask(Converters.convert2CreationTaskData(clusterTaskDO)); + if (ValidateUtils.isNull(createResult) || createResult.failed()) { return ResultStatus.CALL_CLUSTER_TASK_AGENT_FAILED; } try { - clusterTaskDO.setAgentRollbackTaskId(agentTaskId); + clusterTaskDO.setAgentRollbackTaskId(createResult.getData()); if (clusterTaskDao.updateRollback(clusterTaskDO) <= 0) { return ResultStatus.MYSQL_ERROR; } - abstractAgent.actionTask(clusterTaskDO.getAgentTaskId(), ClusterTaskActionEnum.CANCEL.getMessage()); + abstractAgent.actionTask(clusterTaskDO.getAgentTaskId(), ClusterTaskActionEnum.CANCEL); return ResultStatus.SUCCESS; } catch (Exception e) { LOGGER.error("create cluster task failed, clusterTaskDO:{}.", clusterTaskDO, e); @@ -191,11 +191,11 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { return Result.buildFrom(ResultStatus.TASK_NOT_EXIST); } - String stdoutLog = abstractAgent.getTaskLog(getActiveAgentTaskId(clusterTaskDO, hostname), hostname); - if (ValidateUtils.isNull(stdoutLog)) { + Result stdoutLogResult = abstractAgent.getTaskLog(getActiveAgentTaskId(clusterTaskDO, hostname), hostname); + if (ValidateUtils.isNull(stdoutLogResult) || stdoutLogResult.failed()) { return Result.buildFrom(ResultStatus.CALL_CLUSTER_TASK_AGENT_FAILED); } - return new Result<>(stdoutLog); + return new Result<>(stdoutLogResult.getData().getStdout()); } @Override @@ -205,24 +205,33 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { return Result.buildFrom(ResultStatus.TASK_NOT_EXIST); } + Result statusEnumResult = abstractAgent.getTaskExecuteState(getActiveAgentTaskId(clusterTaskDO)); + if (ValidateUtils.isNull(statusEnumResult) || statusEnumResult.failed()) { + return new Result<>(statusEnumResult.getCode(), statusEnumResult.getMessage()); + } + return new Result<>(new ClusterTaskStatus( clusterTaskDO.getId(), clusterTaskDO.getClusterId(), inRollback(clusterTaskDO), - abstractAgent.getTaskState(getActiveAgentTaskId(clusterTaskDO)), + statusEnumResult.getData(), getTaskSubStatus(clusterTaskDO) )); } @Override public ClusterTaskStateEnum getTaskState(Long agentTaskId) { - return abstractAgent.getTaskState(agentTaskId); + Result statusEnumResult = abstractAgent.getTaskExecuteState(agentTaskId); + if (ValidateUtils.isNull(statusEnumResult) || statusEnumResult.failed()) { + return null; + } + return statusEnumResult.getData(); } private List getTaskSubStatus(ClusterTaskDO clusterTaskDO) { Map statusMap = this.getClusterTaskSubState(clusterTaskDO); if (ValidateUtils.isNull(statusMap)) { - return null; + return Collections.emptyList(); } List pauseList = ListUtils.string2StrList(clusterTaskDO.getPauseHostList()); @@ -242,20 +251,22 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { } private Map getClusterTaskSubState(ClusterTaskDO clusterTaskDO) { - Map statusMap = abstractAgent.getTaskResult(clusterTaskDO.getAgentTaskId()); - if (ValidateUtils.isNull(statusMap)) { + Result> statusMapResult = abstractAgent.getTaskResult(clusterTaskDO.getAgentTaskId()); + if (ValidateUtils.isNull(statusMapResult) || statusMapResult.failed()) { return null; } + Map statusMap = statusMapResult.getData(); if (!inRollback(clusterTaskDO)) { return statusMap; } - Map rollbackStatusMap = + Result> rollbackStatusMapResult = abstractAgent.getTaskResult(clusterTaskDO.getAgentRollbackTaskId()); - if (ValidateUtils.isNull(rollbackStatusMap)) { + if (ValidateUtils.isNull(rollbackStatusMapResult) || rollbackStatusMapResult.failed()) { return null; } - statusMap.putAll(rollbackStatusMap); + + statusMap.putAll(rollbackStatusMapResult.getData()); return statusMap; } @@ -276,7 +287,7 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { } catch (Exception e) { LOGGER.error("get all cluster task failed."); } - return null; + return Collections.emptyList(); } @Override @@ -302,9 +313,6 @@ public class ClusterTaskServiceImpl implements ClusterTaskService { } private boolean inRollback(ClusterTaskDO clusterTaskDO) { - if (ClusterTaskConstant.INVALID_AGENT_TASK_ID.equals(clusterTaskDO.getAgentRollbackTaskId())) { - return false; - } - return true; + return !ClusterTaskConstant.INVALID_AGENT_TASK_ID.equals(clusterTaskDO.getAgentRollbackTaskId()); } } \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/KafkaFileServiceImpl.java b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/KafkaFileServiceImpl.java index 307c486c..bef2fb89 100644 --- a/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/KafkaFileServiceImpl.java +++ b/kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/impl/KafkaFileServiceImpl.java @@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; import java.util.ArrayList; import java.util.List; @@ -163,7 +164,7 @@ public class KafkaFileServiceImpl implements KafkaFileService { } @Override - public Result downloadKafkaConfigFile(Long fileId) { + public Result downloadKafkaFile(Long fileId) { KafkaFileDO kafkaFileDO = kafkaFileDao.getById(fileId); if (ValidateUtils.isNull(kafkaFileDO)) { return Result.buildFrom(ResultStatus.RESOURCE_NOT_EXIST); diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java index 4fe01e22..02a11497 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java @@ -50,7 +50,7 @@ public class GatewayHeartbeatController { doList = JsonUtils.parseTopicConnections(clusterId, jsonObject, System.currentTimeMillis()); } catch (Exception e) { LOGGER.error("class=GatewayHeartbeatController||method=receiveTopicConnections||clusterId={}||brokerId={}||msg=parse data failed||exception={}", clusterId, brokerId, e.getMessage()); - return Result.buildFailure("fail"); + return Result.buildGatewayFailure("fail"); } topicConnectionService.batchAdd(doList); diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayServiceDiscoveryController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayServiceDiscoveryController.java index e490368d..425eba75 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayServiceDiscoveryController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayServiceDiscoveryController.java @@ -31,7 +31,6 @@ import java.util.Map; @RestController @RequestMapping(ApiPrefix.GATEWAY_API_V1_PREFIX) public class GatewayServiceDiscoveryController { - private final static Logger LOGGER = LoggerFactory.getLogger(GatewayHeartbeatController.class); @Autowired @@ -65,7 +64,7 @@ public class GatewayServiceDiscoveryController { KafkaBootstrapServerConfig config = gatewayConfigService.getKafkaBootstrapServersConfig(Long.MIN_VALUE); if (ValidateUtils.isNull(config) || ValidateUtils.isNull(config.getClusterIdBootstrapServersMap())) { - return Result.buildFailure("call init kafka bootstrap servers failed"); + return Result.buildGatewayFailure("call init kafka bootstrap servers failed"); } if (ValidateUtils.isEmptyMap(config.getClusterIdBootstrapServersMap())) { return Result.buildSuc(); @@ -81,7 +80,7 @@ public class GatewayServiceDiscoveryController { KafkaBootstrapServerConfig config = gatewayConfigService.getKafkaBootstrapServersConfig(versionNumber); if (ValidateUtils.isNull(config) || ValidateUtils.isNull(config.getClusterIdBootstrapServersMap())) { - return Result.buildFailure("call update kafka bootstrap servers failed"); + return Result.buildGatewayFailure("call update kafka bootstrap servers failed"); } if (ValidateUtils.isEmptyMap(config.getClusterIdBootstrapServersMap())) { return Result.buildSuc(); @@ -99,7 +98,7 @@ public class GatewayServiceDiscoveryController { public Result getMaxRequestNum(@RequestParam("versionNumber") long versionNumber) { RequestQueueConfig config = gatewayConfigService.getRequestQueueConfig(versionNumber); if (ValidateUtils.isNull(config)) { - return Result.buildFailure("call get request queue size config failed"); + return Result.buildGatewayFailure("call get request queue size config failed"); } if (ValidateUtils.isNull(config.getMaxRequestQueueSize())) { return Result.buildSuc(); @@ -119,7 +118,7 @@ public class GatewayServiceDiscoveryController { public Result getAppIdRate(@RequestParam("versionNumber") long versionNumber) { AppRateConfig config = gatewayConfigService.getAppRateConfig(versionNumber); if (ValidateUtils.isNull(config)) { - return Result.buildFailure("call get app rate config failed"); + return Result.buildGatewayFailure("call get app rate config failed"); } if (ValidateUtils.isNull(config.getAppRateLimit())) { return Result.buildSuc(); @@ -139,7 +138,7 @@ public class GatewayServiceDiscoveryController { public Result getIpRate(@RequestParam("versionNumber") long versionNumber) { IpRateConfig config = gatewayConfigService.getIpRateConfig(versionNumber); if (ValidateUtils.isNull(config)) { - return Result.buildFailure("call get ip rate config failed"); + return Result.buildGatewayFailure("call get ip rate config failed"); } if (ValidateUtils.isNull(config.getIpRateLimit())) { return Result.buildSuc(); @@ -160,7 +159,7 @@ public class GatewayServiceDiscoveryController { SpRateConfig config = gatewayConfigService.getSpRateConfig(versionNumber); if (ValidateUtils.isNull(config) || ValidateUtils.isNull(config.getSpRateMap())) { - return Result.buildFailure("call update kafka bootstrap servers failed"); + return Result.buildGatewayFailure("call update kafka bootstrap servers failed"); } if (ValidateUtils.isEmptyMap(config.getSpRateMap())) { return Result.buildSuc(); diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpClusterController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpClusterController.java index 21547aa9..2caaa69b 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpClusterController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpClusterController.java @@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.op; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.ControllerPreferredCandidateDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.rd.ClusterDTO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.service.service.ClusterService; @@ -13,6 +14,7 @@ import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; + /** * @author zengqiao * @date 20/4/23 @@ -25,48 +27,56 @@ public class OpClusterController { private ClusterService clusterService; @ApiOperation(value = "接入集群") - @RequestMapping(value = "clusters", method = RequestMethod.POST) + @PostMapping(value = "clusters") @ResponseBody public Result addNew(@RequestBody ClusterDTO dto) { if (ValidateUtils.isNull(dto) || !dto.legal()) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } return Result.buildFrom( - clusterService.addNew( - ClusterModelConverter.convert2ClusterDO(dto), - SpringTool.getUserName() - ) + clusterService.addNew(ClusterModelConverter.convert2ClusterDO(dto), SpringTool.getUserName()) ); } @ApiOperation(value = "删除集群") - @RequestMapping(value = "clusters", method = RequestMethod.DELETE) + @DeleteMapping(value = "clusters") @ResponseBody public Result delete(@RequestParam(value = "clusterId") Long clusterId) { - return Result.buildFrom(clusterService.deleteById(clusterId)); + return Result.buildFrom(clusterService.deleteById(clusterId, SpringTool.getUserName())); } @ApiOperation(value = "修改集群信息") - @RequestMapping(value = "clusters", method = RequestMethod.PUT) + @PutMapping(value = "clusters") @ResponseBody public Result modify(@RequestBody ClusterDTO reqObj) { if (ValidateUtils.isNull(reqObj) || !reqObj.legal() || ValidateUtils.isNull(reqObj.getClusterId())) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } - ResultStatus rs = clusterService.updateById( - ClusterModelConverter.convert2ClusterDO(reqObj), - SpringTool.getUserName() + return Result.buildFrom( + clusterService.updateById(ClusterModelConverter.convert2ClusterDO(reqObj), SpringTool.getUserName()) ); - return Result.buildFrom(rs); } @ApiOperation(value = "开启|关闭集群监控") - @RequestMapping(value = "clusters/{clusterId}/monitor", method = RequestMethod.PUT) + @PutMapping(value = "clusters/{clusterId}/monitor") @ResponseBody - public Result modifyStatus(@PathVariable Long clusterId, - @RequestParam("status") Integer status) { + public Result modifyStatus(@PathVariable Long clusterId, @RequestParam("status") Integer status) { return Result.buildFrom( clusterService.modifyStatus(clusterId, status, SpringTool.getUserName()) ); } + + @ApiOperation(value = "增加Controller优先候选的Broker", notes = "滴滴内部引擎特性") + @PostMapping(value = "cluster-controller/preferred-candidates") + @ResponseBody + public Result addControllerPreferredCandidates(@RequestBody ControllerPreferredCandidateDTO dto) { + return clusterService.addControllerPreferredCandidates(dto.getClusterId(), dto.getBrokerIdList()); + } + + @ApiOperation(value = "删除Controller优先候选的Broker", notes = "滴滴内部引擎特性") + @DeleteMapping(value = "cluster-controller/preferred-candidates") + @ResponseBody + public Result deleteControllerPreferredCandidates(@RequestBody ControllerPreferredCandidateDTO dto) { + return clusterService.deleteControllerPreferredCandidates(dto.getClusterId(), dto.getBrokerIdList()); + } } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java index a97bb386..66eb3b7e 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java @@ -3,8 +3,11 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.op; import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionAddGatewayConfigDTO; import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionDeleteGatewayConfigDTO; import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionModifyGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.common.bizenum.gateway.GatewayConfigKeyEnum; +import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.service.service.gateway.GatewayConfigService; import com.xiaojukeji.kafka.manager.web.converters.GatewayModelConverter; @@ -16,12 +19,20 @@ import org.springframework.web.bind.annotation.*; @Api(tags = "OP-Gateway配置相关接口(REST)") @RestController +@RequestMapping(ApiPrefix.API_V1_OP_PREFIX) public class OpGatewayConfigController { @Autowired private GatewayConfigService gatewayConfigService; + @ApiOperation(value = "Gateway配置类型", notes = "") + @GetMapping(value = "gateway-configs/type-enums") + @ResponseBody + public Result getClusterModesEnum() { + return new Result<>(JsonUtils.toJson(GatewayConfigKeyEnum.class)); + } + @ApiOperation(value = "创建Gateway配置", notes = "") - @RequestMapping(value = "gateway-configs", method = RequestMethod.POST) + @PostMapping(value = "gateway-configs") @ResponseBody public Result createGatewayConfig(@RequestBody OrderExtensionAddGatewayConfigDTO dto) { if (ValidateUtils.isNull(dto) || !dto.legal()) { @@ -31,7 +42,7 @@ public class OpGatewayConfigController { } @ApiOperation(value = "修改Gateway配置", notes = "") - @RequestMapping(value = "gateway-configs", method = RequestMethod.PUT) + @PutMapping(value = "gateway-configs") @ResponseBody public Result modifyGatewayConfig(@RequestBody OrderExtensionModifyGatewayConfigDTO dto) { if (ValidateUtils.isNull(dto) || !dto.legal()) { @@ -41,7 +52,7 @@ public class OpGatewayConfigController { } @ApiOperation(value = "删除Gateway配置", notes = "") - @RequestMapping(value = "gateway-configs", method = RequestMethod.DELETE) + @DeleteMapping(value = "gateway-configs") @ResponseBody public Result deleteGatewayConfig(@RequestBody OrderExtensionDeleteGatewayConfigDTO dto) { if (ValidateUtils.isNull(dto) || !dto.legal()) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdAccountController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdAccountController.java index 5efbbfce..2ca29082 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdAccountController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdAccountController.java @@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.rd; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.vo.common.AccountVO; +import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.web.converters.AccountConverter; @@ -46,7 +47,7 @@ public class RdAccountController { @RequestMapping(value = "accounts", method = RequestMethod.DELETE) @ResponseBody public Result deleteAccount(@RequestParam("username") String username) { - ResultStatus rs = accountService.deleteByName(username); + ResultStatus rs = accountService.deleteByName(username, SpringTool.getUserName()); return Result.buildFrom(rs); } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java index 3748c3ca..6a46ff0a 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java @@ -1,5 +1,6 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.rd; +import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO; import com.xiaojukeji.kafka.manager.common.entity.vo.rd.GatewayConfigVO; @@ -15,12 +16,13 @@ import java.util.List; @Api(tags = "RD-Gateway配置相关接口(REST)") @RestController +@RequestMapping(ApiPrefix.API_V1_RD_PREFIX) public class RdGatewayConfigController { @Autowired private GatewayConfigService gatewayConfigService; @ApiOperation(value = "Gateway相关配置信息", notes = "") - @RequestMapping(value = "gateway-configs", method = RequestMethod.GET) + @GetMapping(value = "gateway-configs") @ResponseBody public Result> getGatewayConfigs() { List doList = gatewayConfigService.list(); diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdKafkaFileController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdKafkaFileController.java index 823bbe70..eaab7dc9 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdKafkaFileController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdKafkaFileController.java @@ -15,9 +15,16 @@ import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.web.converters.KafkaFileConverter; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import org.apache.tomcat.util.http.fileupload.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; +import java.io.InputStream; +import java.net.URLEncoder; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,6 +37,8 @@ import java.util.Map; @RestController @RequestMapping(ApiPrefix.API_V1_RD_PREFIX) public class RdKafkaFileController { + private final static Logger LOGGER = LoggerFactory.getLogger(RdKafkaFileController.class); + @Autowired private ClusterService clusterService; @@ -71,9 +80,33 @@ public class RdKafkaFileController { return new Result<>(KafkaFileConverter.convertKafkaFileVOList(kafkaFileDOList, clusterService)); } - @ApiOperation(value = "文件预览", notes = "") + @Deprecated + @ApiOperation(value = "文件下载", notes = "") @RequestMapping(value = "kafka-files/{fileId}/config-files", method = RequestMethod.GET) - public Result previewKafkaFile(@PathVariable("fileId") Long fileId) { - return kafkaFileService.downloadKafkaConfigFile(fileId); + public Result downloadKafkaFile(@PathVariable("fileId") Long fileId, HttpServletResponse response) { + Result multipartFileResult = kafkaFileService.downloadKafkaFile(fileId); + + if (multipartFileResult.failed() || ValidateUtils.isNull(multipartFileResult.getData())) { + return multipartFileResult; + } + + InputStream is = null; + try { + response.setContentType(multipartFileResult.getData().getContentType()); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(multipartFileResult.getData().getOriginalFilename(), "UTF-8")); + is = multipartFileResult.getData().getInputStream(); + IOUtils.copy(is, response.getOutputStream()); + } catch (Exception e) { + LOGGER.error("class=RdKafkaFileController||method=downloadKafkaFile||fileId={}||errMsg={}||msg=modify response failed", fileId, e.getMessage()); + } finally { + try { + if (is != null) { + is.close(); + } + } catch (Exception e) { + } + } + return Result.buildSuc(); } } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdOperateRecordController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdOperateRecordController.java index 11f063e6..68068f97 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdOperateRecordController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdOperateRecordController.java @@ -24,14 +24,13 @@ import java.util.List; @RestController @RequestMapping(ApiPrefix.API_V1_RD_PREFIX) public class RdOperateRecordController { - private static final int MAX_RECORD_COUNT = 200; @Autowired private OperateRecordService operateRecordService; @ApiOperation(value = "查询操作记录", notes = "") - @RequestMapping(value = "operate-record", method = RequestMethod.POST) + @PostMapping(value = "operate-record") @ResponseBody public Result> geOperateRecords(@RequestBody OperateRecordDTO dto) { if (ValidateUtils.isNull(dto) || !dto.legal()) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java index f032e921..6a8b5f79 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java @@ -67,6 +67,7 @@ public class GatewayModelConverter { vo.setName(configDO.getName()); vo.setValue(configDO.getValue()); vo.setVersion(configDO.getVersion()); + vo.setDescription(configDO.getDescription()); vo.setCreateTime(configDO.getCreateTime()); vo.setModifyTime(configDO.getModifyTime()); voList.add(vo); @@ -76,18 +77,20 @@ public class GatewayModelConverter { public static GatewayConfigDO convert2GatewayConfigDO(OrderExtensionAddGatewayConfigDTO configDTO) { GatewayConfigDO configDO = new GatewayConfigDO(); - configDO.setType(configDO.getType()); - configDO.setName(configDO.getName()); - configDO.setValue(configDO.getValue()); + configDO.setType(configDTO.getType()); + configDO.setName(configDTO.getName()); + configDO.setValue(configDTO.getValue()); + configDO.setDescription(ValidateUtils.isNull(configDTO.getDescription())? "": configDTO.getDescription()); return configDO; } public static GatewayConfigDO convert2GatewayConfigDO(OrderExtensionModifyGatewayConfigDTO configDTO) { GatewayConfigDO configDO = new GatewayConfigDO(); - configDO.setId(configDO.getId()); - configDO.setType(configDO.getType()); - configDO.setName(configDO.getName()); - configDO.setValue(configDO.getValue()); + configDO.setId(configDTO.getId()); + configDO.setType(configDTO.getType()); + configDO.setName(configDTO.getName()); + configDO.setValue(configDTO.getValue()); + configDO.setDescription(ValidateUtils.isNull(configDTO.getDescription())? "": configDTO.getDescription()); return configDO; } } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/inteceptor/WebMetricsInterceptor.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/inteceptor/WebMetricsInterceptor.java index bf8bc1e1..576fe036 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/inteceptor/WebMetricsInterceptor.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/inteceptor/WebMetricsInterceptor.java @@ -119,7 +119,7 @@ public class WebMetricsInterceptor { ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); String uri = attributes.getRequest().getRequestURI(); if (uri.contains(ApiPrefix.GATEWAY_API_V1_PREFIX)) { - return Result.buildFailure("api limited"); + return Result.buildGatewayFailure("api limited"); } return new Result<>(ResultStatus.OPERATION_FORBIDDEN); } diff --git a/kafka-manager-web/src/main/resources/application.yml b/kafka-manager-web/src/main/resources/application.yml index 8c137da8..008cc048 100644 --- a/kafka-manager-web/src/main/resources/application.yml +++ b/kafka-manager-web/src/main/resources/application.yml @@ -11,6 +11,7 @@ spring: name: kafkamanager datasource: kafka-manager: + jdbc-url: jdbc:mysql://127.0.0.1:3306/logi_kafka_manager?characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2B8 username: admin password: admin @@ -31,7 +32,7 @@ logging: custom: idc: cn jmx: - max-conn: 10 + max-conn: 10 # 2.3版本配置不在这个地方生效 store-metrics-task: community: broker-metrics-enabled: true @@ -52,8 +53,11 @@ account: kcm: enabled: false - storage: - base-url: http://127.0.0.1 + s3: + endpoint: s3.didiyunapi.com + access-key: 1234567890 + secret-key: 0987654321 + bucket: logi-kafka n9e: base-url: http://127.0.0.1:8004 user-token: 12345678 @@ -79,3 +83,15 @@ notify: topic-name: didi-kafka-notify order: detail-url: http://127.0.0.1 + +ldap: + enabled: false + url: ldap://127.0.0.1:389/ + basedn: dc=tsign,dc=cn + factory: com.sun.jndi.ldap.LdapCtxFactory + security: + authentication: simple + principal: cn=admin,dc=tsign,dc=cn + credentials: admin + auth-user-registration-role: normal + auth-user-registration: true diff --git a/pom.xml b/pom.xml index 7165880e..d4165a85 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ - 2.2.0-SNAPSHOT + 2.3.0-SNAPSHOT 2.7.0 1.5.13 @@ -223,6 +223,12 @@ curator-recipes 2.10.0 + + + io.minio + minio + 7.1.0 + \ No newline at end of file