From f551674860f9361b9a43ca5d21af1416cb5f0187 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Mon, 26 Apr 2021 11:16:53 +0800 Subject: [PATCH 01/64] =?UTF-8?q?=E5=88=9B=E5=BB=BAtopic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/entity/dto/normal/TopicAddDTO.java | 43 +++++++++++++++++++ .../manager/service/service/TopicService.java | 6 +++ .../service/impl/TopicServiceImpl.java | 25 +++++++++++ .../normal/NormalTopicController.java | 13 +++++- 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java new file mode 100644 index 00000000..0db12952 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java @@ -0,0 +1,43 @@ +package com.xiaojukeji.kafka.manager.common.entity.dto.normal; + + +import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel(description = "创建topic") +public class TopicAddDTO extends ClusterTopicDTO { + + @ApiModelProperty(value = "appId") + private String appId; + + @ApiModelProperty(value = "峰值流量") + private Long peakBytesIn; + + @ApiModelProperty(value = "备注信息") + private String description; + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public Long getPeakBytesIn() { + return peakBytesIn; + } + + public void setPeakBytesIn(Long peakBytesIn) { + this.peakBytesIn = peakBytesIn; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index dacba4b0..39516fad 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -4,6 +4,7 @@ import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ao.*; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; @@ -105,4 +106,9 @@ public interface TopicService { List getTopicBrokerList(Long clusterId, String topicName); Result checkTopicOffsetChanged(Long physicalClusterId, String topicName, Long latestTime); + + /** + * 创建topic + */ + Result addTopic(TopicAddDTO dto); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 5dea0561..1e2e66d3 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -3,6 +3,7 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.constant.Constant; @@ -20,6 +21,7 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.PartitionMap; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.PartitionState; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; import com.xiaojukeji.kafka.manager.dao.TopicAppMetricsDao; +import com.xiaojukeji.kafka.manager.dao.TopicDao; import com.xiaojukeji.kafka.manager.dao.TopicMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicRequestMetricsDao; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; @@ -87,6 +89,9 @@ public class TopicServiceImpl implements TopicService { @Autowired private AbstractHealthScoreStrategy healthScoreStrategy; + @Autowired + private TopicDao topicDao; + @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { try { @@ -824,6 +829,26 @@ public class TopicServiceImpl implements TopicService { return new Result<>(TopicOffsetChangedEnum.UNKNOWN); } + @Override + public Result addTopic(TopicAddDTO dto) { + TopicDO topicDO = topicManagerService.getByTopicName(dto.getClusterId(), dto.getTopicName()); + if (!ValidateUtils.isNull(topicDO)) { + // 该topic已存在 + return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); + } + TopicDO topic = new TopicDO(); + topic.setAppId(dto.getAppId()); + topic.setClusterId(dto.getClusterId()); + topic.setDescription(dto.getDescription()); + topic.setTopicName(dto.getTopicName()); + topic.setPeakBytesIn(dto.getPeakBytesIn() * 1024 * 1024); + int insert = topicDao.insert(topic); + if (insert > 0) { + return Result.buildFrom(ResultStatus.SUCCESS); + } + return Result.buildFrom(ResultStatus.FAIL); + } + private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 6e59816b..f1b0b091 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -6,6 +6,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; @@ -335,8 +336,18 @@ public class NormalTopicController { } return new Result<>(TopicModelConverter.convert2TopicMineAppVOList( - topicManagerService.getTopicMineApps(physicalClusterId, topicName, SpringTool.getUserName())) + topicManagerService.getTopicMineApps(physicalClusterId, topicName, SpringTool.getUserName())) ); } + @ApiOperation(value = "创建topic",notes = "创建topic") + @RequestMapping(value = {"/topics/add"},method = RequestMethod.POST) + @ResponseBody + public Result addTopic(@RequestBody TopicAddDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.addTopic(dto); + } + } \ No newline at end of file From 7fa1a66f7e355257223ea3aa8cbc31ac260486bd Mon Sep 17 00:00:00 2001 From: HuYueeer <40758588+Huyueeer@users.noreply.github.com> Date: Mon, 26 Apr 2021 14:51:12 +0800 Subject: [PATCH 02/64] =?UTF-8?q?=E6=9B=B4=E6=96=B0Topic=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/dev_guide/dynamic_config_manager.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/dev_guide/dynamic_config_manager.md b/docs/dev_guide/dynamic_config_manager.md index 0965cb48..0d57b01e 100644 --- a/docs/dev_guide/dynamic_config_manager.md +++ b/docs/dev_guide/dynamic_config_manager.md @@ -119,4 +119,24 @@ TOPIC_INSUFFICIENT_PARTITION_CONFIG 50 ] } -``` \ No newline at end of file +``` +## 4、专家服务——Topic资源治理 + +判定规则描述你们完善一下！！！ + +Topic分区不足相关的动态配置(页面在运维管控->平台管理->配置管理)： + +配置Key： +``` +EXPIRED_TOPIC_CONFIG +``` + +配置Value： +```json +{ + "minExpiredDay": 30, #过期时间大于此值才显示 + "ignoreClusterIdList": [ # 忽略的集群 + 50 + ] +} +``` From ee1ab30c2c7e832061b2f49d39c85606c6adf922 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Mon, 26 Apr 2021 17:48:32 +0800 Subject: [PATCH 03/64] =?UTF-8?q?=E5=88=9B=E5=BB=BAtopic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/service/impl/TopicServiceImpl.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 1e2e66d3..590081b5 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -14,6 +14,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata; @@ -25,6 +26,7 @@ import com.xiaojukeji.kafka.manager.dao.TopicDao; import com.xiaojukeji.kafka.manager.dao.TopicMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicRequestMetricsDao; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; +import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.cache.KafkaClientPool; import com.xiaojukeji.kafka.manager.service.cache.KafkaMetricsCache; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; @@ -43,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; import java.util.*; @@ -92,6 +95,9 @@ public class TopicServiceImpl implements TopicService { @Autowired private TopicDao topicDao; + @Autowired + private AuthorityDao authorityDao; + @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { try { @@ -829,6 +835,7 @@ public class TopicServiceImpl implements TopicService { return new Result<>(TopicOffsetChangedEnum.UNKNOWN); } + @Transactional @Override public Result addTopic(TopicAddDTO dto) { TopicDO topicDO = topicManagerService.getByTopicName(dto.getClusterId(), dto.getTopicName()); @@ -836,6 +843,14 @@ public class TopicServiceImpl implements TopicService { // 该topic已存在 return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); } + // 给创建topic的用户该topic权限 + AuthorityDO authorityDO = new AuthorityDO(); + authorityDO.setAppId(dto.getAppId()); + authorityDO.setClusterId(dto.getClusterId()); + authorityDO.setTopicName(dto.getTopicName()); + authorityDO.setAccess(3); + authorityDao.insert(authorityDO); + // 记录该topic TopicDO topic = new TopicDO(); topic.setAppId(dto.getAppId()); topic.setClusterId(dto.getClusterId()); From b5901a281990fd77a5754bff917803b591b6c618 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Wed, 28 Apr 2021 11:08:01 +0800 Subject: [PATCH 04/64] =?UTF-8?q?=E5=88=9B=E5=BB=BAtopic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/TopicServiceImpl.java | 74 +++++++++++-------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 590081b5..6bbd2e7b 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -1,8 +1,11 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; +import com.xiaojukeji.kafka.manager.common.constant.SystemCodeConstant; +import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.ao.config.CreateTopicElemConfig; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; @@ -14,7 +17,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics; -import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; +import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata; @@ -22,11 +25,9 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.PartitionMap; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.PartitionState; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; import com.xiaojukeji.kafka.manager.dao.TopicAppMetricsDao; -import com.xiaojukeji.kafka.manager.dao.TopicDao; import com.xiaojukeji.kafka.manager.dao.TopicMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicRequestMetricsDao; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; -import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.cache.KafkaClientPool; import com.xiaojukeji.kafka.manager.service.cache.KafkaMetricsCache; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; @@ -45,7 +46,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; import java.util.*; @@ -93,10 +93,10 @@ public class TopicServiceImpl implements TopicService { private AbstractHealthScoreStrategy healthScoreStrategy; @Autowired - private TopicDao topicDao; + private ConfigService configService; @Autowired - private AuthorityDao authorityDao; + private AdminService adminService; @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { @@ -835,36 +835,46 @@ public class TopicServiceImpl implements TopicService { return new Result<>(TopicOffsetChangedEnum.UNKNOWN); } - @Transactional @Override public Result addTopic(TopicAddDTO dto) { - TopicDO topicDO = topicManagerService.getByTopicName(dto.getClusterId(), dto.getTopicName()); - if (!ValidateUtils.isNull(topicDO)) { - // 该topic已存在 - return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); - } - // 给创建topic的用户该topic权限 - AuthorityDO authorityDO = new AuthorityDO(); - authorityDO.setAppId(dto.getAppId()); - authorityDO.setClusterId(dto.getClusterId()); - authorityDO.setTopicName(dto.getTopicName()); - authorityDO.setAccess(3); - authorityDao.insert(authorityDO); - // 记录该topic - TopicDO topic = new TopicDO(); - topic.setAppId(dto.getAppId()); - topic.setClusterId(dto.getClusterId()); - topic.setDescription(dto.getDescription()); - topic.setTopicName(dto.getTopicName()); - topic.setPeakBytesIn(dto.getPeakBytesIn() * 1024 * 1024); - int insert = topicDao.insert(topic); - if (insert > 0) { - return Result.buildFrom(ResultStatus.SUCCESS); - } - return Result.buildFrom(ResultStatus.FAIL); + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + TopicDO topic = topicManagerService.getByTopicName(physicalClusterId, dto.getTopicName()); + if (!ValidateUtils.isNull(topic)) { + return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); + } + CreateTopicElemConfig createTopicConfig = + configService.getCreateTopicConfig(physicalClusterId, SystemCodeConstant.KAFKA_MANAGER); + if (dto.getPeakBytesIn() > createTopicConfig.getAutoExecMaxPeakBytesInUnitB()) { + return Result.buildFailure("流量峰值设置过大，不超过30M"); + } + ClusterDO clusterDO = clusterService.getById(physicalClusterId); + if (ValidateUtils.isNull(clusterDO)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + List brokerList = regionService + .getIdleRegionBrokerList(clusterDO.getId(), createTopicConfig.getRegionIdList()); + if (ValidateUtils.isEmptyList(brokerList)) { + return Result.buildFrom(ResultStatus.BROKER_NOT_EXIST); + } + TopicDO topicDO = new TopicDO(); + topicDO.setAppId(dto.getAppId()); + topicDO.setClusterId(clusterDO.getId()); + topicDO.setTopicName(dto.getTopicName()); + topicDO.setDescription(dto.getDescription()); + topicDO.setPeakBytesIn(dto.getPeakBytesIn()); + Long partitionNum = Math.max(1, dto.getPeakBytesIn() / (3 * 1024 * 1024)); + Properties properties = TopicCreationConstant + .createNewProperties(createTopicConfig.getRetentionTimeUnitHour() * 60 * 60 * 1000L); + ResultStatus rs = adminService.createTopic(clusterDO, topicDO, partitionNum.intValue(), + createTopicConfig.getReplicaNum(), null, brokerList, properties, null, + SpringTool.getUserName()); + return Result.buildFrom(rs); } - private Result checkTopicOffsetChanged(ClusterDO clusterDO, + private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { if (ValidateUtils.isNull(clusterDO) From df3283f526c44b95ce06f1b6d923e551806fafba Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Wed, 28 Apr 2021 11:22:53 +0800 Subject: [PATCH 05/64] =?UTF-8?q?=E5=88=A0=E9=99=A4topic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/service/service/TopicService.java | 5 +++++ .../service/service/impl/TopicServiceImpl.java | 16 +++++++++++++++- .../versionone/normal/NormalTopicController.java | 10 ++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index 39516fad..f15332ff 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -111,4 +111,9 @@ public interface TopicService { * 创建topic */ Result addTopic(TopicAddDTO dto); + + /** + * 删除topic + */ + Result deleteTopic(Long clusterId, String topicName); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 6bbd2e7b..0fffc4d8 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -874,7 +874,21 @@ public class TopicServiceImpl implements TopicService { return Result.buildFrom(rs); } - private Result checkTopicOffsetChanged(ClusterDO clusterDO, + @Override + public Result deleteTopic(Long clusterId, String topicName) { + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(clusterId); + if (ValidateUtils.isNull(physicalClusterId)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + ClusterDO clusterDO = clusterService.getById(physicalClusterId); + if (ValidateUtils.isNull(clusterDO)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + ResultStatus rs = adminService.deleteTopic(clusterDO, topicName, SpringTool.getUserName()); + return Result.buildFrom(rs); + } + + private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { if (ValidateUtils.isNull(clusterDO) diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index f1b0b091..3ce2b163 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -350,4 +350,14 @@ public class NormalTopicController { return topicService.addTopic(dto); } + @ApiOperation(value = "删除topic",notes = "删除topic") + @RequestMapping(value = {"{clusterId}/topics/{topicName}/delete"},method = RequestMethod.DELETE) + @ResponseBody + public Result deleteTopic(@PathVariable Long clusterId, + @PathVariable String topicName) { + if (ValidateUtils.isNull(clusterId) || ValidateUtils.isNull(topicName)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.deleteTopic(clusterId,topicName); + } } \ No newline at end of file From 473fc27b49b3ddb3ac732c276621b0ba82312f8c Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Wed, 28 Apr 2021 18:05:58 +0800 Subject: [PATCH 06/64] =?UTF-8?q?=E5=88=9B=E5=BB=BAtopic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/entity/dto/normal/TopicAddDTO.java | 76 ++++++++++++++++--- .../entity/dto/normal/TopicQuotaDTO.java | 42 ++++++++++ .../manager/service/service/TopicService.java | 6 ++ .../service/impl/TopicServiceImpl.java | 75 +++++++++++------- .../normal/NormalTopicController.java | 11 +++ 5 files changed, 175 insertions(+), 35 deletions(-) create mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java index 0db12952..4783f106 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java @@ -1,22 +1,39 @@ package com.xiaojukeji.kafka.manager.common.entity.dto.normal; - import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import java.util.List; +import java.util.Properties; + @ApiModel(description = "创建topic") public class TopicAddDTO extends ClusterTopicDTO { - @ApiModelProperty(value = "appId") + @ApiModelProperty(value = "AppID") private String appId; - @ApiModelProperty(value = "峰值流量") - private Long peakBytesIn; + @ApiModelProperty(value = "分区数") + private Integer partitionNum; - @ApiModelProperty(value = "备注信息") + @ApiModelProperty(value = "副本数") + private Integer replicaNum; + + @ApiModelProperty(value = "消息保存时间(ms)") + private Long retentionTime; + + @ApiModelProperty(value = "brokerId列表") + private List brokerIdList; + + @ApiModelProperty(value = "RegionId") + private Long regionId; + + @ApiModelProperty(value = "备注") private String description; + @ApiModelProperty(value = "Topic属性列表") + private Properties properties; + public String getAppId() { return appId; } @@ -25,12 +42,44 @@ public class TopicAddDTO extends ClusterTopicDTO { this.appId = appId; } - public Long getPeakBytesIn() { - return peakBytesIn; + public Integer getPartitionNum() { + return partitionNum; } - public void setPeakBytesIn(Long peakBytesIn) { - this.peakBytesIn = peakBytesIn; + public void setPartitionNum(Integer partitionNum) { + this.partitionNum = partitionNum; + } + + public Integer getReplicaNum() { + return replicaNum; + } + + public void setReplicaNum(Integer replicaNum) { + this.replicaNum = replicaNum; + } + + public Long getRetentionTime() { + return retentionTime; + } + + public void setRetentionTime(Long retentionTime) { + this.retentionTime = retentionTime; + } + + public List getBrokerIdList() { + return brokerIdList; + } + + public void setBrokerIdList(List brokerIdList) { + this.brokerIdList = brokerIdList; + } + + public Long getRegionId() { + return regionId; + } + + public void setRegionId(Long regionId) { + this.regionId = regionId; } public String getDescription() { @@ -40,4 +89,13 @@ public class TopicAddDTO extends ClusterTopicDTO { public void setDescription(String description) { this.description = description; } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java new file mode 100644 index 00000000..aeb86814 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java @@ -0,0 +1,42 @@ +package com.xiaojukeji.kafka.manager.common.entity.dto.normal; + +import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel(description = "配额调整") +public class TopicQuotaDTO extends ClusterTopicDTO { + + @ApiModelProperty(value = "appId") + private String appId; + + @ApiModelProperty(value = "发送数据速率") + private Long produceQuota; + + @ApiModelProperty(value = "消费数据速率") + private Long consumeQuota; + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public Long getProduceQuota() { + return produceQuota; + } + + public void setProduceQuota(Long produceQuota) { + this.produceQuota = produceQuota; + } + + public Long getConsumeQuota() { + return consumeQuota; + } + + public void setConsumeQuota(Long consumeQuota) { + this.consumeQuota = consumeQuota; + } +} diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index f15332ff..0a9b14bf 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -7,6 +7,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicMetricsDO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicBrokerDTO; @@ -116,4 +117,9 @@ public interface TopicService { * 删除topic */ Result deleteTopic(Long clusterId, String topicName); + + /** + * 配额调整 + */ + Result addTopicQuota(TopicQuotaDTO dto); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 0fffc4d8..b02a2495 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -1,12 +1,12 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; -import com.xiaojukeji.kafka.manager.common.constant.SystemCodeConstant; -import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; +import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; -import com.xiaojukeji.kafka.manager.common.entity.ao.config.CreateTopicElemConfig; +import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.constant.Constant; @@ -34,6 +34,7 @@ import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; +import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.service.strategy.AbstractHealthScoreStrategy; import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils; import com.xiaojukeji.kafka.manager.service.utils.MetricsConvertUtils; @@ -93,10 +94,10 @@ public class TopicServiceImpl implements TopicService { private AbstractHealthScoreStrategy healthScoreStrategy; @Autowired - private ConfigService configService; + private AdminService adminService; @Autowired - private AdminService adminService; + private QuotaService quotaService; @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { @@ -837,58 +838,80 @@ public class TopicServiceImpl implements TopicService { @Override public Result addTopic(TopicAddDTO dto) { + //获取物理集群id Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); if (ValidateUtils.isNull(physicalClusterId)) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } - TopicDO topic = topicManagerService.getByTopicName(physicalClusterId, dto.getTopicName()); - if (!ValidateUtils.isNull(topic)) { - return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); - } - CreateTopicElemConfig createTopicConfig = - configService.getCreateTopicConfig(physicalClusterId, SystemCodeConstant.KAFKA_MANAGER); - if (dto.getPeakBytesIn() > createTopicConfig.getAutoExecMaxPeakBytesInUnitB()) { - return Result.buildFailure("流量峰值设置过大，不超过30M"); - } + //获取集群信息 ClusterDO clusterDO = clusterService.getById(physicalClusterId); if (ValidateUtils.isNull(clusterDO)) { return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); } - List brokerList = regionService - .getIdleRegionBrokerList(clusterDO.getId(), createTopicConfig.getRegionIdList()); - if (ValidateUtils.isEmptyList(brokerList)) { - return Result.buildFrom(ResultStatus.BROKER_NOT_EXIST); + //判断topic是否存在 + TopicDO topic = topicManagerService.getByTopicName(physicalClusterId, dto.getTopicName()); + if (!ValidateUtils.isNull(topic)) { + return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); } + //构建topicDo TopicDO topicDO = new TopicDO(); topicDO.setAppId(dto.getAppId()); - topicDO.setClusterId(clusterDO.getId()); + topicDO.setClusterId(dto.getClusterId()); topicDO.setTopicName(dto.getTopicName()); topicDO.setDescription(dto.getDescription()); - topicDO.setPeakBytesIn(dto.getPeakBytesIn()); - Long partitionNum = Math.max(1, dto.getPeakBytesIn() / (3 * 1024 * 1024)); - Properties properties = TopicCreationConstant - .createNewProperties(createTopicConfig.getRetentionTimeUnitHour() * 60 * 60 * 1000L); - ResultStatus rs = adminService.createTopic(clusterDO, topicDO, partitionNum.intValue(), - createTopicConfig.getReplicaNum(), null, brokerList, properties, null, + //构建properties + Properties properties = dto.getProperties(); + if (ValidateUtils.isNull(properties)) { + properties = new Properties(); + } + properties.put(KafkaConstant.RETENTION_MS_KEY, String.valueOf(dto.getRetentionTime())); + //创建topic + ResultStatus rs = adminService.createTopic(clusterDO, topicDO, dto.getPartitionNum(), + dto.getReplicaNum(), dto.getRegionId(), dto.getBrokerIdList(), properties, SpringTool.getUserName(), SpringTool.getUserName()); return Result.buildFrom(rs); } @Override public Result deleteTopic(Long clusterId, String topicName) { + //获得物理集群id Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(clusterId); if (ValidateUtils.isNull(physicalClusterId)) { return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); } + //获取集群信息 ClusterDO clusterDO = clusterService.getById(physicalClusterId); if (ValidateUtils.isNull(clusterDO)) { return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); } + //删除topic ResultStatus rs = adminService.deleteTopic(clusterDO, topicName, SpringTool.getUserName()); return Result.buildFrom(rs); } - private Result checkTopicOffsetChanged(ClusterDO clusterDO, + @Override + public Result addTopicQuota(TopicQuotaDTO dto) { + //获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //构建配额 + TopicQuota topicQuota = new TopicQuota(); + topicQuota.setClusterId(physicalClusterId); + topicQuota.setAppId(dto.getAppId()); + topicQuota.setTopicName(dto.getTopicName()); + topicQuota.setProduceQuota(dto.getProduceQuota()); + topicQuota.setConsumeQuota(dto.getConsumeQuota()); + //配额调整 + int result = quotaService.addTopicQuota(topicQuota); + if (result > 0) { + return Result.buildFrom(ResultStatus.SUCCESS); + } + return Result.buildFrom(ResultStatus.FAIL); + } + + private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { if (ValidateUtils.isNull(clusterDO) diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 3ce2b163..75d97e68 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -8,6 +8,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.TopicBusinessInfoVO; @@ -360,4 +361,14 @@ public class NormalTopicController { } return topicService.deleteTopic(clusterId,topicName); } + + @ApiOperation(value = "配额调整",notes = "配额调整") + @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) + @ResponseBody + public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.addTopicQuota(dto); + } } \ No newline at end of file From ca0ebe0d758dffb4d9224351b75b45e9c70c002c Mon Sep 17 00:00:00 2001 From: zengqiao Date: Sat, 8 May 2021 11:15:25 +0800 Subject: [PATCH 07/64] =?UTF-8?q?faq=E4=B8=AD=E8=A1=A5=E5=85=85=E8=AF=B4?= =?UTF-8?q?=E6=98=8Eheartbeat=E8=A1=A8=E7=9A=84=E4=BD=9C=E7=94=A8=E5=8F=8A?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E5=88=A0=E9=99=A4=E6=98=AF=E5=90=A6=E5=BD=B1?= =?UTF-8?q?=E5=93=8D=E7=89=A9=E7=90=86=E9=9B=86=E7=BE=A4=E7=9A=84=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/faq.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 5b0b103d..25eeec2a 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -23,6 +23,8 @@ - 11、连接信息、耗时信息、磁盘信息为什么没有数据？ - 12、逻辑集群申请审批通过之后为什么看不到逻辑集群？ - 13、heartbeat表关联业务和使用场景是什么？ +- 14、集群的删除，是否会真正的删除集群？ + --- ### 0、支持哪些Kafka版本？ @@ -129,3 +131,17 @@ 逻辑集群的申请与审批仅仅只是一个工单流程，并不会去实际创建逻辑集群，逻辑集群的创建还需要手动去创建。 具体的操作可见：[kafka-manager 接入集群](add_cluster/add_cluster.md)。 + + +### 13、heartbeat表关联业务和使用场景是什么？ + +做任务抢占用的。 + +KM支持HA的方式部署，那么部署多台的时候，就会出现每一台都可能去做指标收集的事情，这块就使用heartbeat表做KM的存活性判断，然后进行任务的抢占或者是均衡。 + +更多详细的内容，可以看一下源码中，heartbeat表在哪里被使用了。 + + +### 14、集群的删除，是否会真正的删除集群？ + +Logi-KM的运维管控，集群列表中的集群删除，仅仅只是将该集群从Logi-KM中进行删除，并不会对真正的物理集群做什么操作。 From f0b3b9f7f421e28ae453ceb5e920cc7dbf3d69f2 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Sat, 8 May 2021 11:23:06 +0800 Subject: [PATCH 08/64] =?UTF-8?q?=E6=89=A9=E5=88=86=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../entity/dto/normal/TopicExpandDTO.java | 57 +++++++++++++++++++ .../manager/service/service/TopicService.java | 6 ++ .../service/impl/TopicServiceImpl.java | 25 +++++++- .../normal/NormalTopicController.java | 12 ++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java new file mode 100644 index 00000000..9dc6d5a1 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java @@ -0,0 +1,57 @@ +package com.xiaojukeji.kafka.manager.common.entity.dto.normal; + +import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel(description = "扩分区") +public class TopicExpandDTO extends ClusterTopicDTO { + + @ApiModelProperty(value = "regionId") + private Long regionId; + + @ApiModelProperty(value = "brokerId列表") + private List brokerIds; + + @ApiModelProperty(value = "新增分区数") + private Integer partitionNum; + + public Long getRegionId() { + return regionId; + } + + public void setRegionId(Long regionId) { + this.regionId = regionId; + } + + public List getBrokerIds() { + return brokerIds; + } + + public void setBrokerIds(List brokerIds) { + this.brokerIds = brokerIds; + } + + public Integer getPartitionNum() { + return partitionNum; + } + + public void setPartitionNum(Integer partitionNum) { + this.partitionNum = partitionNum; + } + + public boolean paramLegal() { + if (ValidateUtils.isNull(clusterId) + || ValidateUtils.isNull(topicName) + || ValidateUtils.isNull(partitionNum) || partitionNum <= 0) { + return false; + } + if (ValidateUtils.isEmptyList(brokerIds) && ValidateUtils.isNull(regionId)) { + return false; + } + return true; + } +} diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index 0a9b14bf..f850d1ad 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -7,6 +7,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicMetricsDO; @@ -122,4 +123,9 @@ public interface TopicService { * 配额调整 */ Result addTopicQuota(TopicQuotaDTO dto); + + /** + * 扩分区 + */ + Result expandTopic(TopicExpandDTO dto); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index b02a2495..8c215d8b 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -6,6 +6,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; @@ -911,7 +912,29 @@ public class TopicServiceImpl implements TopicService { return Result.buildFrom(ResultStatus.FAIL); } - private Result checkTopicOffsetChanged(ClusterDO clusterDO, + @Override + public Result expandTopic(TopicExpandDTO dto) { + // 校验非空 + if (!dto.paramLegal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + //获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + //获取集群信息 + ClusterDO clusterDO = clusterService.getById(physicalClusterId); + if (ValidateUtils.isNull(clusterDO)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //扩分区 + ResultStatus resultStatus = adminService.expandPartitions(clusterDO, dto.getTopicName(), dto.getPartitionNum(), + dto.getRegionId(), dto.getBrokerIds(), SpringTool.getUserName()); + return Result.buildFrom(resultStatus); + } + + private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { if (ValidateUtils.isNull(clusterDO) diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 75d97e68..269529fc 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -8,6 +8,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; @@ -351,6 +352,17 @@ public class NormalTopicController { return topicService.addTopic(dto); } + @ApiOperation(value = "扩分区",notes = "扩分区") + @RequestMapping(value = "{/topics/expand}",method = RequestMethod.POST) + @ResponseBody + public Result expandTopic(@RequestBody TopicExpandDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.expandTopic(dto); + } + + @ApiOperation(value = "删除topic",notes = "删除topic") @RequestMapping(value = {"{clusterId}/topics/{topicName}/delete"},method = RequestMethod.DELETE) @ResponseBody From b390df08b513b06fb6087c6a9f0cab935bed5b7f Mon Sep 17 00:00:00 2001 From: zengqiao Date: Sat, 8 May 2021 11:28:52 +0800 Subject: [PATCH 09/64] =?UTF-8?q?faq=E8=A1=A5=E5=85=85app=E7=9A=84?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/faq.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 25eeec2a..bf569fb2 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -24,6 +24,7 @@ - 12、逻辑集群申请审批通过之后为什么看不到逻辑集群？ - 13、heartbeat表关联业务和使用场景是什么？ - 14、集群的删除，是否会真正的删除集群？ +- 15、APP(应用)如何被使用起来？ --- @@ -145,3 +146,12 @@ KM支持HA的方式部署，那么部署多台的时候，就会出现每一台 ### 14、集群的删除，是否会真正的删除集群？ Logi-KM的运维管控，集群列表中的集群删除，仅仅只是将该集群从Logi-KM中进行删除，并不会对真正的物理集群做什么操作。 + + +### 15、APP(应用)如何被使用起来？ + +app在Logi-KM中可以近似理解为租户，或者是kafka里面的一个账号的概念。 + +界面中显示的app信息、权限信息等，在平台层面仅仅只是控制Topic或集群在平台上的可见性，如果使用的是社区版本的Kafka，那么实际上是不能真正的管控到客户端对Topic的生产和消费。 + +但是如果是使用的滴滴的Kafka-Gateway，那么是可以做到对客户端的生产和消费的权限管控。滴滴的Kafka-Gateway暂未开源，属于企业服务，具体的可以入群交流，群地址在README中。 From ed163a80e0750f0241c37f568822ab62d4d7812d Mon Sep 17 00:00:00 2001 From: zengqiao Date: Sat, 8 May 2021 11:45:27 +0800 Subject: [PATCH 10/64] bump commons-beanutils version to 1.9.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 397bed74..9d4a5d30 100644 --- a/pom.xml +++ b/pom.xml @@ -215,7 +215,7 @@ commons-beanutils commons-beanutils - 1.9.3 + 1.9.4 From e342e646ff893beaa40b24055b24a39f2b793eaa Mon Sep 17 00:00:00 2001 From: zengqiao Date: Sat, 8 May 2021 13:50:04 +0800 Subject: [PATCH 11/64] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9B=A0DB=E6=97=B6?= =?UTF-8?q?=E5=8C=BA=E4=B8=8D=E5=AF=B9=E5=AF=BC=E8=87=B4=E5=91=A8=E6=9C=9F?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E4=B8=8D=E8=83=BD=E8=A2=AB=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../task/component/AbstractScheduledTask.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/component/AbstractScheduledTask.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/component/AbstractScheduledTask.java index b0700975..7eddb926 100644 --- a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/component/AbstractScheduledTask.java +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/component/AbstractScheduledTask.java @@ -148,10 +148,12 @@ public abstract class AbstractScheduledTask implements Sch } Collections.sort(allTaskList); - List hostList = heartbeatDao.selectActiveHosts( - new Date(now - ScheduledTaskConstant.HEARTBEAT_TIME) - ); + List hostList = heartbeatDao.selectActiveHosts(new Date(now - ScheduledTaskConstant.HEARTBEAT_TIME)); if (ValidateUtils.isEmptyList(hostList)) { + // 当前无机器注册，导致周期任务(Topic指标存DB等任务)不可被触发执行。 + // 大概率原因可能是：DB的时区不对，注册的时间错误导致查询不出来。 + // 如果是单台方式部署的Logi-KM，那么也可能是服务新上线，或者是服务不正常导致的。 + LOGGER.error("customScheduled task running, but without registrant, and so scheduled tasks can't execute, scheduledName:{}.", scheduledName); return Lists.newArrayList(); } @@ -163,8 +165,10 @@ public abstract class AbstractScheduledTask implements Sch idx++; } if (idx == hostList.size()) { - // 当前机器未注册 - LOGGER.error("customScheduled task running, registers not conclude, scheduledName:{}.", scheduledName); + // 当前机器未注册, 原因可能是： + // 1、当前服务新上线，确实暂未注册到DB中。 + // 2、当前服务异常，比如进行FGC等，导致注册任务停止了。 + LOGGER.warn("customScheduled task running, registrants not conclude present machine, scheduledName:{}.", scheduledName); return Lists.newArrayList(); } From 9185d2646b78f3e61e98d034d3836cff0a1a48b3 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Mon, 10 May 2021 16:22:58 +0800 Subject: [PATCH 12/64] =?UTF-8?q?=E6=9D=83=E9=99=90=E7=94=B3=E8=AF=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/entity/dto/TopicAuthorityDTO.java | 30 +++++++++++++ .../manager/service/service/TopicService.java | 6 +++ .../service/impl/TopicServiceImpl.java | 45 ++++++++++++++++++- .../normal/NormalTopicController.java | 11 +++++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java new file mode 100644 index 00000000..7be62ff6 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java @@ -0,0 +1,30 @@ +package com.xiaojukeji.kafka.manager.common.entity.dto; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel(description = "权限调整") +public class TopicAuthorityDTO extends ClusterTopicDTO{ + + @ApiModelProperty(value = "appId") + private String appId; + + @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写") + private Integer access; + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public Integer getAccess() { + return access; + } + + public void setAccess(Integer access) { + this.access = access; + } +} diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index f850d1ad..3646d6fa 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -4,6 +4,7 @@ import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ao.*; +import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; @@ -128,4 +129,9 @@ public interface TopicService { * 扩分区 */ Result expandTopic(TopicExpandDTO dto); + + /** + * 权限调整 + */ + Result addAuthorityAdd(TopicAuthorityDTO dto); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 8c215d8b..93fd6d94 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -5,6 +5,7 @@ import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; +import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; @@ -18,6 +19,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant; @@ -29,6 +31,7 @@ import com.xiaojukeji.kafka.manager.dao.TopicAppMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicRequestMetricsDao; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; +import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.cache.KafkaClientPool; import com.xiaojukeji.kafka.manager.service.cache.KafkaMetricsCache; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; @@ -100,6 +103,9 @@ public class TopicServiceImpl implements TopicService { @Autowired private QuotaService quotaService; + @Autowired + private AuthorityDao authorityDao; + @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { try { @@ -909,7 +915,7 @@ public class TopicServiceImpl implements TopicService { if (result > 0) { return Result.buildFrom(ResultStatus.SUCCESS); } - return Result.buildFrom(ResultStatus.FAIL); + return Result.buildFrom(ResultStatus.MYSQL_ERROR); } @Override @@ -934,6 +940,43 @@ public class TopicServiceImpl implements TopicService { return Result.buildFrom(resultStatus); } + @Override + public Result addAuthorityAdd(TopicAuthorityDTO dto) { + //查询该用户拥有的应用 + List appDOs = appService.getByPrincipal(SpringTool.getUserName()); + if (ValidateUtils.isEmptyList(appDOs)) { + //该用户无应用，需要先申请应用 + return Result.buildFrom(ResultStatus.APP_NOT_EXIST); + } + List appIds = appDOs.stream().map(AppDO::getId).collect(Collectors.toList()); + if (!appIds.contains(dto.getAccess())) { + //入参中的appId，该用户未拥有 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + //获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + //集群不存在 + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //获取集群信息 + ClusterDO clusterDO = clusterService.getById(physicalClusterId); + if (ValidateUtils.isNull(clusterDO)) { + //集群不存在 + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //构建authorityDo + AuthorityDO authorityDO = new AuthorityDO(); + authorityDO.setClusterId(physicalClusterId); + authorityDO.setAppId(dto.getAppId()); + authorityDO.setTopicName(dto.getTopicName()); + authorityDO.setAccess(dto.getAccess()); + if (authorityDao.insert(authorityDO) > 0) { + return Result.buildFrom(ResultStatus.SUCCESS); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } + private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 269529fc..7a9ded7d 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -6,6 +6,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; @@ -383,4 +384,14 @@ public class NormalTopicController { } return topicService.addTopicQuota(dto); } + + @ApiOperation(value = "权限调整",notes = "权限调整") + @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) + @ResponseBody + public Result addAuthorityAdd(@RequestBody TopicAuthorityDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.addAuthorityAdd(dto); + } } \ No newline at end of file From f359ff995d0ca8851f6b484541e9286fbf2e2211 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 11 May 2021 08:41:37 +0800 Subject: [PATCH 13/64] =?UTF-8?q?faq=E4=B8=AD=E5=A2=9E=E5=8A=A0=E5=BA=94?= =?UTF-8?q?=E7=94=A8=E4=B8=8B=E7=BA=BF=E5=A4=B1=E8=B4=A5=E7=9A=84=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/faq.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index bf569fb2..42ba747a 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -25,6 +25,7 @@ - 13、heartbeat表关联业务和使用场景是什么？ - 14、集群的删除，是否会真正的删除集群？ - 15、APP(应用)如何被使用起来？ +- 16、为什么下线应用提示operation forbidden？ --- @@ -155,3 +156,17 @@ app在Logi-KM中可以近似理解为租户，或者是kafka里面的一个账 界面中显示的app信息、权限信息等，在平台层面仅仅只是控制Topic或集群在平台上的可见性，如果使用的是社区版本的Kafka，那么实际上是不能真正的管控到客户端对Topic的生产和消费。 但是如果是使用的滴滴的Kafka-Gateway，那么是可以做到对客户端的生产和消费的权限管控。滴滴的Kafka-Gateway暂未开源，属于企业服务，具体的可以入群交流，群地址在README中。 + + +### 16、为什么下线应用提示operation forbidden？ + +**原因一：** + +该应用还存在对Topic的权限，因此导致下线失败。具体查看的位置在"Topic管理-》应用管理-》详情"，可以看到应用对哪些Topic还有权限。 + +只有当权限全部去除之后，才可以下线应用。 + +**原因二：** + +使用的是2.4.0之前的旧版本，旧版本存在缓存更新的BUG，建议升级至最新的版本，或者简单粗暴的就是重启一下KM。 + From a43b04a98b6509f92c35d5c1cc874e39535b0f9f Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 11 May 2021 10:23:25 +0800 Subject: [PATCH 14/64] =?UTF-8?q?=E5=A4=84=E7=90=86=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../normal/NormalTopicController.java | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 7a9ded7d..809a3032 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -6,11 +6,9 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.TopicBusinessInfoVO; @@ -363,35 +361,4 @@ public class NormalTopicController { return topicService.expandTopic(dto); } - - @ApiOperation(value = "删除topic",notes = "删除topic") - @RequestMapping(value = {"{clusterId}/topics/{topicName}/delete"},method = RequestMethod.DELETE) - @ResponseBody - public Result deleteTopic(@PathVariable Long clusterId, - @PathVariable String topicName) { - if (ValidateUtils.isNull(clusterId) || ValidateUtils.isNull(topicName)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.deleteTopic(clusterId,topicName); - } - - @ApiOperation(value = "配额调整",notes = "配额调整") - @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) - @ResponseBody - public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.addTopicQuota(dto); - } - - @ApiOperation(value = "权限调整",notes = "权限调整") - @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) - @ResponseBody - public Result addAuthorityAdd(@RequestBody TopicAuthorityDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.addAuthorityAdd(dto); - } } \ No newline at end of file From 9334e9552ff2e120e535fab3327c5fe8803b40b2 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 11 May 2021 10:25:25 +0800 Subject: [PATCH 15/64] =?UTF-8?q?=E5=A4=84=E7=90=86=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../normal/NormalTopicController.java | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 809a3032..86755eca 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -6,9 +6,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.TopicBusinessInfoVO; @@ -341,24 +339,4 @@ public class NormalTopicController { ); } - @ApiOperation(value = "创建topic",notes = "创建topic") - @RequestMapping(value = {"/topics/add"},method = RequestMethod.POST) - @ResponseBody - public Result addTopic(@RequestBody TopicAddDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.addTopic(dto); - } - - @ApiOperation(value = "扩分区",notes = "扩分区") - @RequestMapping(value = "{/topics/expand}",method = RequestMethod.POST) - @ResponseBody - public Result expandTopic(@RequestBody TopicExpandDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.expandTopic(dto); - } - } \ No newline at end of file From 696657c09e0e7d194d1a3a9a2653a56a4099f249 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 11 May 2021 10:40:58 +0800 Subject: [PATCH 16/64] =?UTF-8?q?=E5=A4=84=E7=90=86=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../normal/NormalTopicController.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index aaac290f..a742d487 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -6,7 +6,11 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.TopicBusinessInfoVO; @@ -360,4 +364,54 @@ public class NormalTopicController { return new Result<>(new TopicStatisticMetricsVO(maxAvgBytesIn)); } + @ApiOperation(value = "创建topic",notes = "创建topic") + @RequestMapping(value = {"/topics/add"},method = RequestMethod.POST) + @ResponseBody + public Result addTopic(@RequestBody TopicAddDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.addTopic(dto); + } + + @ApiOperation(value = "扩分区",notes = "扩分区") + @RequestMapping(value = "{/topics/expand}",method = RequestMethod.POST) + @ResponseBody + public Result expandTopic(@RequestBody TopicExpandDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.expandTopic(dto); + } + + @ApiOperation(value = "删除topic",notes = "删除topic") + @RequestMapping(value = {"{clusterId}/topics/{topicName}/delete"},method = RequestMethod.DELETE) + @ResponseBody + public Result deleteTopic(@PathVariable Long clusterId, + @PathVariable String topicName) { + if (ValidateUtils.isNull(clusterId) || ValidateUtils.isNull(topicName)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.deleteTopic(clusterId,topicName); + } + + @ApiOperation(value = "配额调整",notes = "配额调整") + @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) + @ResponseBody + public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.addTopicQuota(dto); + } + + @ApiOperation(value = "权限调整",notes = "权限调整") + @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) + @ResponseBody + public Result addAuthorityAdd(@RequestBody TopicAuthorityDTO dto) { + if (ValidateUtils.isNull(dto)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return topicService.addAuthorityAdd(dto); + } } \ No newline at end of file From b68ba0bff6f560ba1e00832f7b292cdbf1082e85 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 11 May 2021 16:58:44 +0800 Subject: [PATCH 17/64] =?UTF-8?q?=E9=85=8D=E9=A2=9D=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/entity/dto/TopicAuthorityDTO.java | 30 ----- .../common/entity/dto/normal/TopicAddDTO.java | 101 --------------- .../entity/dto/normal/TopicExpandDTO.java | 57 --------- .../manager/service/service/TopicService.java | 22 ---- .../service/impl/TopicServiceImpl.java | 118 ------------------ .../normal/NormalTopicController.java | 43 ------- 6 files changed, 371 deletions(-) delete mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java delete mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java delete mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java deleted file mode 100644 index 7be62ff6..00000000 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/TopicAuthorityDTO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.xiaojukeji.kafka.manager.common.entity.dto; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; - -@ApiModel(description = "权限调整") -public class TopicAuthorityDTO extends ClusterTopicDTO{ - - @ApiModelProperty(value = "appId") - private String appId; - - @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写") - private Integer access; - - public String getAppId() { - return appId; - } - - public void setAppId(String appId) { - this.appId = appId; - } - - public Integer getAccess() { - return access; - } - - public void setAccess(Integer access) { - this.access = access; - } -} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java deleted file mode 100644 index 4783f106..00000000 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicAddDTO.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.xiaojukeji.kafka.manager.common.entity.dto.normal; - -import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; - -import java.util.List; -import java.util.Properties; - -@ApiModel(description = "创建topic") -public class TopicAddDTO extends ClusterTopicDTO { - - @ApiModelProperty(value = "AppID") - private String appId; - - @ApiModelProperty(value = "分区数") - private Integer partitionNum; - - @ApiModelProperty(value = "副本数") - private Integer replicaNum; - - @ApiModelProperty(value = "消息保存时间(ms)") - private Long retentionTime; - - @ApiModelProperty(value = "brokerId列表") - private List brokerIdList; - - @ApiModelProperty(value = "RegionId") - private Long regionId; - - @ApiModelProperty(value = "备注") - private String description; - - @ApiModelProperty(value = "Topic属性列表") - private Properties properties; - - public String getAppId() { - return appId; - } - - public void setAppId(String appId) { - this.appId = appId; - } - - public Integer getPartitionNum() { - return partitionNum; - } - - public void setPartitionNum(Integer partitionNum) { - this.partitionNum = partitionNum; - } - - public Integer getReplicaNum() { - return replicaNum; - } - - public void setReplicaNum(Integer replicaNum) { - this.replicaNum = replicaNum; - } - - public Long getRetentionTime() { - return retentionTime; - } - - public void setRetentionTime(Long retentionTime) { - this.retentionTime = retentionTime; - } - - public List getBrokerIdList() { - return brokerIdList; - } - - public void setBrokerIdList(List brokerIdList) { - this.brokerIdList = brokerIdList; - } - - public Long getRegionId() { - return regionId; - } - - public void setRegionId(Long regionId) { - this.regionId = regionId; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Properties getProperties() { - return properties; - } - - public void setProperties(Properties properties) { - this.properties = properties; - } - -} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java deleted file mode 100644 index 9dc6d5a1..00000000 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicExpandDTO.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.xiaojukeji.kafka.manager.common.entity.dto.normal; - -import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; -import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; - -import java.util.List; - -@ApiModel(description = "扩分区") -public class TopicExpandDTO extends ClusterTopicDTO { - - @ApiModelProperty(value = "regionId") - private Long regionId; - - @ApiModelProperty(value = "brokerId列表") - private List brokerIds; - - @ApiModelProperty(value = "新增分区数") - private Integer partitionNum; - - public Long getRegionId() { - return regionId; - } - - public void setRegionId(Long regionId) { - this.regionId = regionId; - } - - public List getBrokerIds() { - return brokerIds; - } - - public void setBrokerIds(List brokerIds) { - this.brokerIds = brokerIds; - } - - public Integer getPartitionNum() { - return partitionNum; - } - - public void setPartitionNum(Integer partitionNum) { - this.partitionNum = partitionNum; - } - - public boolean paramLegal() { - if (ValidateUtils.isNull(clusterId) - || ValidateUtils.isNull(topicName) - || ValidateUtils.isNull(partitionNum) || partitionNum <= 0) { - return false; - } - if (ValidateUtils.isEmptyList(brokerIds) && ValidateUtils.isNull(regionId)) { - return false; - } - return true; - } -} diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index 3646d6fa..cacfb9f0 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -4,11 +4,8 @@ import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ao.*; -import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicMetricsDO; @@ -110,28 +107,9 @@ public interface TopicService { Result checkTopicOffsetChanged(Long physicalClusterId, String topicName, Long latestTime); - /** - * 创建topic - */ - Result addTopic(TopicAddDTO dto); - - /** - * 删除topic - */ - Result deleteTopic(Long clusterId, String topicName); - /** * 配额调整 */ Result addTopicQuota(TopicQuotaDTO dto); - /** - * 扩分区 - */ - Result expandTopic(TopicExpandDTO dto); - - /** - * 权限调整 - */ - Result addAuthorityAdd(TopicAuthorityDTO dto); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 5e17386a..f6140cc9 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -1,13 +1,9 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; -import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; -import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; @@ -19,8 +15,6 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics; -import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; -import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConstant; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata; @@ -843,59 +837,6 @@ public class TopicServiceImpl implements TopicService { return new Result<>(TopicOffsetChangedEnum.UNKNOWN); } - @Override - public Result addTopic(TopicAddDTO dto) { - //获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - //获取集群信息 - ClusterDO clusterDO = clusterService.getById(physicalClusterId); - if (ValidateUtils.isNull(clusterDO)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //判断topic是否存在 - TopicDO topic = topicManagerService.getByTopicName(physicalClusterId, dto.getTopicName()); - if (!ValidateUtils.isNull(topic)) { - return Result.buildFrom(ResultStatus.TOPIC_ALREADY_EXIST); - } - //构建topicDo - TopicDO topicDO = new TopicDO(); - topicDO.setAppId(dto.getAppId()); - topicDO.setClusterId(dto.getClusterId()); - topicDO.setTopicName(dto.getTopicName()); - topicDO.setDescription(dto.getDescription()); - //构建properties - Properties properties = dto.getProperties(); - if (ValidateUtils.isNull(properties)) { - properties = new Properties(); - } - properties.put(KafkaConstant.RETENTION_MS_KEY, String.valueOf(dto.getRetentionTime())); - //创建topic - ResultStatus rs = adminService.createTopic(clusterDO, topicDO, dto.getPartitionNum(), - dto.getReplicaNum(), dto.getRegionId(), dto.getBrokerIdList(), properties, SpringTool.getUserName(), - SpringTool.getUserName()); - return Result.buildFrom(rs); - } - - @Override - public Result deleteTopic(Long clusterId, String topicName) { - //获得物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(clusterId); - if (ValidateUtils.isNull(physicalClusterId)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //获取集群信息 - ClusterDO clusterDO = clusterService.getById(physicalClusterId); - if (ValidateUtils.isNull(clusterDO)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //删除topic - ResultStatus rs = adminService.deleteTopic(clusterDO, topicName, SpringTool.getUserName()); - return Result.buildFrom(rs); - } - @Override public Result addTopicQuota(TopicQuotaDTO dto) { //获取物理集群id @@ -918,65 +859,6 @@ public class TopicServiceImpl implements TopicService { return Result.buildFrom(ResultStatus.MYSQL_ERROR); } - @Override - public Result expandTopic(TopicExpandDTO dto) { - // 校验非空 - if (!dto.paramLegal()) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - //获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - //获取集群信息 - ClusterDO clusterDO = clusterService.getById(physicalClusterId); - if (ValidateUtils.isNull(clusterDO)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //扩分区 - ResultStatus resultStatus = adminService.expandPartitions(clusterDO, dto.getTopicName(), dto.getPartitionNum(), - dto.getRegionId(), dto.getBrokerIds(), SpringTool.getUserName()); - return Result.buildFrom(resultStatus); - } - - @Override - public Result addAuthorityAdd(TopicAuthorityDTO dto) { - //查询该用户拥有的应用 - List appDOs = appService.getByPrincipal(SpringTool.getUserName()); - if (ValidateUtils.isEmptyList(appDOs)) { - //该用户无应用，需要先申请应用 - return Result.buildFrom(ResultStatus.APP_NOT_EXIST); - } - List appIds = appDOs.stream().map(AppDO::getId).collect(Collectors.toList()); - if (!appIds.contains(dto.getAccess())) { - //入参中的appId，该用户未拥有 - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - //获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - //集群不存在 - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //获取集群信息 - ClusterDO clusterDO = clusterService.getById(physicalClusterId); - if (ValidateUtils.isNull(clusterDO)) { - //集群不存在 - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //构建authorityDo - AuthorityDO authorityDO = new AuthorityDO(); - authorityDO.setClusterId(physicalClusterId); - authorityDO.setAppId(dto.getAppId()); - authorityDO.setTopicName(dto.getTopicName()); - authorityDO.setAccess(dto.getAccess()); - if (authorityDao.insert(authorityDO) > 0) { - return Result.buildFrom(ResultStatus.SUCCESS); - } - return Result.buildFrom(ResultStatus.MYSQL_ERROR); - } - private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index a742d487..0b6dfde6 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -6,10 +6,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.TopicAuthorityDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicAddDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicExpandDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; @@ -364,37 +361,6 @@ public class NormalTopicController { return new Result<>(new TopicStatisticMetricsVO(maxAvgBytesIn)); } - @ApiOperation(value = "创建topic",notes = "创建topic") - @RequestMapping(value = {"/topics/add"},method = RequestMethod.POST) - @ResponseBody - public Result addTopic(@RequestBody TopicAddDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.addTopic(dto); - } - - @ApiOperation(value = "扩分区",notes = "扩分区") - @RequestMapping(value = "{/topics/expand}",method = RequestMethod.POST) - @ResponseBody - public Result expandTopic(@RequestBody TopicExpandDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.expandTopic(dto); - } - - @ApiOperation(value = "删除topic",notes = "删除topic") - @RequestMapping(value = {"{clusterId}/topics/{topicName}/delete"},method = RequestMethod.DELETE) - @ResponseBody - public Result deleteTopic(@PathVariable Long clusterId, - @PathVariable String topicName) { - if (ValidateUtils.isNull(clusterId) || ValidateUtils.isNull(topicName)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.deleteTopic(clusterId,topicName); - } - @ApiOperation(value = "配额调整",notes = "配额调整") @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) @ResponseBody @@ -405,13 +371,4 @@ public class NormalTopicController { return topicService.addTopicQuota(dto); } - @ApiOperation(value = "权限调整",notes = "权限调整") - @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) - @ResponseBody - public Result addAuthorityAdd(@RequestBody TopicAuthorityDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.addAuthorityAdd(dto); - } } \ No newline at end of file From eb8fe7758235e09da72cf2d7fadf2c597d43c6b9 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Wed, 12 May 2021 11:31:41 +0800 Subject: [PATCH 18/64] =?UTF-8?q?=E9=85=8D=E9=A2=9D=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../entity/dto/normal/TopicQuotaDTO.java | 10 ++++ .../manager/service/service/TopicService.java | 6 -- .../service/impl/TopicServiceImpl.java | 35 ----------- .../normal/NormalTopicController.java | 11 ---- .../thirdpart/ThirdPartTopicController.java | 58 ++++++++++++++++++- 5 files changed, 67 insertions(+), 53 deletions(-) diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java index aeb86814..99399624 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java @@ -1,6 +1,7 @@ package com.xiaojukeji.kafka.manager.common.entity.dto.normal; import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -39,4 +40,13 @@ public class TopicQuotaDTO extends ClusterTopicDTO { public void setConsumeQuota(Long consumeQuota) { this.consumeQuota = consumeQuota; } + + public boolean paramLegal() { + if (ValidateUtils.isNull(clusterId) + || ValidateUtils.isNull(topicName) + || ValidateUtils.isNull(appId)) { + return false; + } + return true; + } } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java index cacfb9f0..9e4c244c 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicService.java @@ -6,7 +6,6 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ao.*; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.*; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicMetricsDO; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicBrokerDTO; @@ -107,9 +106,4 @@ public interface TopicService { Result checkTopicOffsetChanged(Long physicalClusterId, String topicName, Long latestTime); - /** - * 配额调整 - */ - Result addTopicQuota(TopicQuotaDTO dto); - } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index f6140cc9..63191888 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -3,8 +3,6 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; -import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.constant.Constant; @@ -25,14 +23,12 @@ import com.xiaojukeji.kafka.manager.dao.TopicAppMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicMetricsDao; import com.xiaojukeji.kafka.manager.dao.TopicRequestMetricsDao; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; -import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.cache.KafkaClientPool; import com.xiaojukeji.kafka.manager.service.cache.KafkaMetricsCache; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; -import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.service.strategy.AbstractHealthScoreStrategy; import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils; import com.xiaojukeji.kafka.manager.service.utils.MetricsConvertUtils; @@ -91,15 +87,6 @@ public class TopicServiceImpl implements TopicService { @Autowired private AbstractHealthScoreStrategy healthScoreStrategy; - @Autowired - private AdminService adminService; - - @Autowired - private QuotaService quotaService; - - @Autowired - private AuthorityDao authorityDao; - @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { try { @@ -837,28 +824,6 @@ public class TopicServiceImpl implements TopicService { return new Result<>(TopicOffsetChangedEnum.UNKNOWN); } - @Override - public Result addTopicQuota(TopicQuotaDTO dto) { - //获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //构建配额 - TopicQuota topicQuota = new TopicQuota(); - topicQuota.setClusterId(physicalClusterId); - topicQuota.setAppId(dto.getAppId()); - topicQuota.setTopicName(dto.getTopicName()); - topicQuota.setProduceQuota(dto.getProduceQuota()); - topicQuota.setConsumeQuota(dto.getConsumeQuota()); - //配额调整 - int result = quotaService.addTopicQuota(topicQuota); - if (result > 0) { - return Result.buildFrom(ResultStatus.SUCCESS); - } - return Result.buildFrom(ResultStatus.MYSQL_ERROR); - } - private Result checkTopicOffsetChanged(ClusterDO clusterDO, String topicName, Map endOffsetMap) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java index 0b6dfde6..aaac290f 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java @@ -7,7 +7,6 @@ import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.TopicBusinessInfoVO; @@ -361,14 +360,4 @@ public class NormalTopicController { return new Result<>(new TopicStatisticMetricsVO(maxAvgBytesIn)); } - @ApiOperation(value = "配额调整",notes = "配额调整") - @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) - @ResponseBody - public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { - if (ValidateUtils.isNull(dto)) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return topicService.addTopicQuota(dto); - } - } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index b247cdb8..ee39b39a 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -5,7 +5,10 @@ import com.xiaojukeji.kafka.manager.common.constant.Constant; import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; +import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadataVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupVO; @@ -15,9 +18,12 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; +import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; +import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; +import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter; import com.xiaojukeji.kafka.manager.web.converters.TopicModelConverter; @@ -52,6 +58,15 @@ public class ThirdPartTopicController { @Autowired private TopicManagerService topicManagerService; + @Autowired + private AuthorityService authorityService; + + @Autowired + private QuotaService quotaService; + + @Autowired + private LogicalClusterMetadataManager logicalClusterMetadataManager; + @ApiOperation(value = "Topic元信息", notes = "LogX调用") @RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET) @ResponseBody @@ -133,4 +148,45 @@ public class ThirdPartTopicController { topicManagerService.getTopicAuthorizedApps(physicalClusterId, topicName)) ); } -} \ No newline at end of file + + @ApiOperation(value = "配额调整",notes = "配额调整") + @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) + @ResponseBody + public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { + //非空校验 + if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + //获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //权限判断(access 0:无权限, 1:读, 2:写, 3:读写) + AuthorityDO authority = authorityService.getAuthority(physicalClusterId, dto.getTopicName(), dto.getAppId()); + if (ValidateUtils.isNull(authority) || authority.getAccess() == 0) { + return Result.buildFrom(ResultStatus.USER_WITHOUT_AUTHORITY); + } + if (authority.getAccess() == 1) { + //可以消费 + dto.setProduceQuota(null); + } + if (authority.getAccess() == 2) { + //可以生产 + dto.setConsumeQuota(null); + } + //构建topicquota + TopicQuota topicQuotaDO = new TopicQuota(); + topicQuotaDO.setAppId(dto.getAppId()); + topicQuotaDO.setClusterId(physicalClusterId); + topicQuotaDO.setTopicName(dto.getTopicName()); + topicQuotaDO.setConsumeQuota(dto.getConsumeQuota()); + topicQuotaDO.setProduceQuota(dto.getProduceQuota()); + //添加配额 + if (quotaService.addTopicQuota(topicQuotaDO) > 0) { + return Result.buildFrom(ResultStatus.SUCCESS); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } + +} From 07c32732472734341d971972dee23e358de77d20 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Wed, 12 May 2021 20:16:47 +0800 Subject: [PATCH 19/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0v2.4.0=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Releases_Notes.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Releases_Notes.md b/Releases_Notes.md index 46b5753e..471ce227 100644 --- a/Releases_Notes.md +++ b/Releases_Notes.md @@ -7,6 +7,33 @@ --- +## v2.4.0 + +版本上线时间：2021-04-26 + + +### 能力提升 + +- 增加App与Topic自动化审批开关 +- Broker元信息中增加Rack信息 +- 升级MySQL 驱动，支持MySQL 8+ +- 增加操作记录查询界面 + +### 体验优化 + +- FAQ告警组说明优化 +- 用户手册共享及 独享集群概念优化 +- 用户管理界面，前端限制用户删除自己 + +### bug修复 + +- 修复op-util类中创建Topic失败的接口 +- 周期同步Topic到DB的任务修复，将Topic列表查询从缓存调整为直接查DB +- 应用下线审批失败的功能修复，将权限为0(无权限)的数据进行过滤 +- 修复登录及权限绕过的漏洞 +- 修复研发角色展示接入集群、暂停监控等按钮的问题 + + ## v2.3.0 版本上线时间：2021-02-08 From 9286761c30c32f666c0444276d01490f9279fde1 Mon Sep 17 00:00:00 2001 From: EricZeng Date: Thu, 13 May 2021 10:26:07 +0800 Subject: [PATCH 20/64] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BD=93=E9=AA=8C?= =?UTF-8?q?=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新体验地址 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index db8a1b52..e69e1688 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ 滴滴Logi-KafkaManager脱胎于滴滴内部多年的Kafka运营实践经验，是面向Kafka用户、Kafka运维人员打造的共享多租户Kafka云平台。专注于Kafka运维管控、监控告警、资源治理等核心场景，经历过大规模集群、海量大数据的考验。内部满意度高达90%的同时，还与多家知名企业达成商业化合作。 ### 1.1 快速体验地址 -- 体验地址 http://117.51.146.109:8080 账号密码 admin/admin + +- 体验地址 http://117.51.150.133:8080 账号密码 admin/admin ### 1.2 体验地图 相比较于同类产品的用户视角单一（大多为管理员视角），滴滴Logi-KafkaManager建立了基于分角色、多场景视角的体验地图。分别是：**用户体验地图、运维体验地图、运营体验地图** From f88a14ac0abed22a30b821e2de49ea041e9482df Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 13 May 2021 16:22:05 +0800 Subject: [PATCH 21/64] =?UTF-8?q?=E8=A1=A5=E5=85=85=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E4=B8=BA=E4=BB=80=E4=B9=88=E5=88=A0=E9=99=A4Topic=E6=88=90?= =?UTF-8?q?=E5=8A=9F=E4=B9=8B=E5=90=8E,=20=E5=8F=88=E7=AB=8B=E9=A9=AC?= =?UTF-8?q?=E5=87=BA=E7=8E=B0=E7=9A=84=E5=8E=9F=E5=9B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/faq.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 42ba747a..54ac5543 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -26,6 +26,7 @@ - 14、集群的删除，是否会真正的删除集群？ - 15、APP(应用)如何被使用起来？ - 16、为什么下线应用提示operation forbidden？ +- 17、删除Topic成功，为什么过一会儿之后又出现了？ --- @@ -170,3 +171,27 @@ app在Logi-KM中可以近似理解为租户，或者是kafka里面的一个账 使用的是2.4.0之前的旧版本，旧版本存在缓存更新的BUG，建议升级至最新的版本，或者简单粗暴的就是重启一下KM。 + +### 17、删除Topic成功，为什么过一会儿之后又出现了？ + +**原因说明：** + +Logi-KM会去请求Topic的endoffset信息，要获取这个信息就需要发送metadata请求，发送metadata请求的时候，如果集群允许自动创建Topic，那么当Topic不存在时，就会自动将该Topic创建出来。 + + +**问题解决：** + +因为在Logi-KM上，禁止Kafka客户端内部元信息获取这个动作非常的难做到，因此短时间内这个问题不好从Logi-KM上解决。 + +当然，对于不存在的Topic，Logi-KM是不会进行元信息请求的，因此也不用担心会莫名其妙的创建一个Topic出来。 + +但是，另外一点，对于开启允许Topic自动创建的集群，建议是关闭该功能，开启是非常危险的，如果关闭之后，Logi-KM也不会有这个问题。 + +最后这里举个开启这个配置后，非常危险的代码例子吧： + +```java +for (int i= 0; i < 100000; ++i) { + // 如果是客户端类似这样写的，那么一启动，那么将创建10万个Topic出来，集群元信息瞬间爆炸，controller可能就不可服务了。 + producer.send(new ProducerRecord("logi_km" + i,"hello logi_km")); +} +``` From 7218aaf52e1d8f8032140d53106466b8647af30e Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Thu, 13 May 2021 17:49:42 +0800 Subject: [PATCH 22/64] =?UTF-8?q?=E6=9D=83=E9=99=90=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openapi/common/dto/TopicAuthorityDTO.java | 43 ++++++++++++++ .../thirdpart/ThirdPartTopicController.java | 56 +++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java diff --git a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java new file mode 100644 index 00000000..564b31d6 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java @@ -0,0 +1,43 @@ +package com.xiaojukeji.kafka.manager.openapi.common.dto; + +import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel(description = "权限调整") +public class TopicAuthorityDTO extends ClusterTopicDTO { + + @ApiModelProperty(value = "appId") + private String appId; + + @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写") + private Integer access; + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public Integer getAccess() { + return access; + } + + public void setAccess(Integer access) { + this.access = access; + } + + @Override + public boolean paramLegal() { + if (ValidateUtils.isNull(clusterId) + || ValidateUtils.isNull(topicName) + || ValidateUtils.isNull(appId) + || ValidateUtils.isNull(access)) { + return false; + } + return true; + } +} diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index ee39b39a..f2c86fe0 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -8,13 +8,17 @@ import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; +import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadataVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicAuthorizedAppVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicRequestTimeDetailVO; +import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; +import com.xiaojukeji.kafka.manager.openapi.common.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; @@ -22,6 +26,7 @@ import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; +import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; @@ -35,6 +40,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.stream.Collectors; /** * @author zengqiao @@ -67,6 +73,9 @@ public class ThirdPartTopicController { @Autowired private LogicalClusterMetadataManager logicalClusterMetadataManager; + @Autowired + private AppService appService; + @ApiOperation(value = "Topic元信息", notes = "LogX调用") @RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET) @ResponseBody @@ -189,4 +198,51 @@ public class ThirdPartTopicController { return Result.buildFrom(ResultStatus.MYSQL_ERROR); } + @ApiOperation(value = "权限调整",notes = "权限调整") + @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) + @ResponseBody + public Result addAuthority(@RequestBody TopicAuthorityDTO dto) { + //非空校验 + if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + //查询该用户拥有的应用 + List appDOs = appService.getByPrincipal(SpringTool.getUserName()); + if (ValidateUtils.isEmptyList(appDOs)) { + //该用户无应用，需要先申请应用 + return Result.buildFrom(ResultStatus.APP_NOT_EXIST); + } + List appIds = appDOs.stream().map(AppDO::getId).collect(Collectors.toList()); + if (!appIds.contains(dto.getAccess())) { + //入参中的appId，该用户未拥有 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + //获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + //集群不存在 + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //获取集群信息 + ClusterDO clusterDO = clusterService.getById(physicalClusterId); + if (ValidateUtils.isNull(clusterDO)) { + //集群不存在 + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + TopicDO topic = topicManagerService.getByTopicName(physicalClusterId, dto.getTopicName()); + if (ValidateUtils.isNull(topic)) { + //topic不存在 + return Result.buildFrom(ResultStatus.TOPIC_NOT_EXIST); + } + //构建authorityDo + AuthorityDO authorityDO = new AuthorityDO(); + authorityDO.setClusterId(physicalClusterId); + authorityDO.setAppId(dto.getAppId()); + authorityDO.setTopicName(dto.getTopicName()); + authorityDO.setAccess(dto.getAccess()); + if (authorityService.addAuthority(authorityDO) > 0) { + return Result.buildFrom(ResultStatus.SUCCESS); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } } From 960017280d43cd0bb69ce7adbaeeb68608353a4d Mon Sep 17 00:00:00 2001 From: EricZeng Date: Mon, 17 May 2021 09:57:56 +0800 Subject: [PATCH 23/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Topic=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E7=9A=84=E9=85=8D=E7=BD=AE=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加Topic资源治理的配置说明 --- docs/dev_guide/dynamic_config_manager.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev_guide/dynamic_config_manager.md b/docs/dev_guide/dynamic_config_manager.md index 0d57b01e..aa042a6d 100644 --- a/docs/dev_guide/dynamic_config_manager.md +++ b/docs/dev_guide/dynamic_config_manager.md @@ -122,7 +122,7 @@ TOPIC_INSUFFICIENT_PARTITION_CONFIG ``` ## 4、专家服务——Topic资源治理 -判定规则描述你们完善一下！！！ +首先，我们认为在一定的时间长度内，Topic的分区offset没有任何变化的Topic，即没有数据写入的Topic，为过期的Topic。 Topic分区不足相关的动态配置(页面在运维管控->平台管理->配置管理)： From 28a72513195f282aca6332a1682f6adf3e902cc6 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Mon, 17 May 2021 15:31:08 +0800 Subject: [PATCH 24/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=9B=91=E6=8E=A7?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/{alarm_rules.md => monitor_desc.md} | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) rename docs/user_guide/{alarm_rules.md => monitor_desc.md} (75%) diff --git a/docs/user_guide/alarm_rules.md b/docs/user_guide/monitor_desc.md similarity index 75% rename from docs/user_guide/alarm_rules.md rename to docs/user_guide/monitor_desc.md index 57cba628..abd06209 100644 --- a/docs/user_guide/alarm_rules.md +++ b/docs/user_guide/monitor_desc.md @@ -4,11 +4,16 @@ --- +## 报警策略-监控指标说明 + +| 指标 | 含义 |备注 | +| --- | --- | --- | +| online-kafka-consumer-lag | 消费时，按照分区的维度进行监控lag数 | lag表示有多少数据没有被消费，因为按照分区的维度监控，所以告警时一般会有分区信息 | +| online-kafka-consumer-maxLag | 消费时，按照整个Topic的维度，监控Topic所有的分区里面的那个最大的lag | 比如每个分区的lag分别是3、5、7，那么maxLag的值就是max(3,5,7)=7 | +| online-kafka-consumer-maxDelayTime | 消费时，按照Topic维度监控预计的消费延迟 | 这块是按照lag和messagesIn之间的关系计算出来的，可能会有误差 | ## 报警策略-报警函数介绍 - - | 类别 | 函数 | 含义 |函数文案 |备注 | | --- | --- | --- | --- | --- | | 发生次数 |all，n | 最近$n个周期内，全发生 | 连续发生(all) | | From 8b95b3ffc755d53d94402aecdb71024af0f92fdb Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 18 May 2021 11:25:41 +0800 Subject: [PATCH 25/64] =?UTF-8?q?=E9=85=8D=E9=A2=9D=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/entity/ao/gateway/TopicQuota.java | 13 +++ .../{normal => gateway}/TopicQuotaDTO.java | 2 +- .../thirdpart/ThirdPartTopicController.java | 88 +------------------ 3 files changed, 18 insertions(+), 85 deletions(-) rename kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/{normal => gateway}/TopicQuotaDTO.java (94%) diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/gateway/TopicQuota.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/gateway/TopicQuota.java index 7b3bc979..6b734348 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/gateway/TopicQuota.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/gateway/TopicQuota.java @@ -1,5 +1,7 @@ package com.xiaojukeji.kafka.manager.common.entity.ao.gateway; +import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.TopicQuotaDTO; + /** * @author zhongyuankai * @date 2020/4/27 @@ -65,4 +67,15 @@ public class TopicQuota { ", consumeQuota=" + consumeQuota + '}'; } + + public static TopicQuota buildFrom(TopicQuotaDTO dto) { + TopicQuota topicQuota = new TopicQuota(); + topicQuota.setAppId(dto.getAppId()); + topicQuota.setClusterId(dto.getClusterId()); + topicQuota.setTopicName(dto.getTopicName()); + topicQuota.setProduceQuota(dto.getProduceQuota()); + topicQuota.setConsumeQuota(dto.getConsumeQuota()); + return topicQuota; + } + } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java similarity index 94% rename from kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java rename to kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java index 99399624..dd95a3ba 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/normal/TopicQuotaDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.kafka.manager.common.entity.dto.normal; +package com.xiaojukeji.kafka.manager.common.entity.dto.gateway; import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index f2c86fe0..a1940e4c 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -6,19 +6,14 @@ import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; -import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicQuotaDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; -import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; -import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; -import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadataVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicAuthorizedAppVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicRequestTimeDetailVO; -import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; -import com.xiaojukeji.kafka.manager.openapi.common.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; @@ -26,8 +21,6 @@ import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; -import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; -import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter; @@ -40,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; -import java.util.stream.Collectors; /** * @author zengqiao @@ -64,18 +56,12 @@ public class ThirdPartTopicController { @Autowired private TopicManagerService topicManagerService; - @Autowired - private AuthorityService authorityService; - @Autowired private QuotaService quotaService; @Autowired private LogicalClusterMetadataManager logicalClusterMetadataManager; - @Autowired - private AppService appService; - @ApiOperation(value = "Topic元信息", notes = "LogX调用") @RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET) @ResponseBody @@ -171,78 +157,12 @@ public class ThirdPartTopicController { if (ValidateUtils.isNull(physicalClusterId)) { return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); } - //权限判断(access 0:无权限, 1:读, 2:写, 3:读写) - AuthorityDO authority = authorityService.getAuthority(physicalClusterId, dto.getTopicName(), dto.getAppId()); - if (ValidateUtils.isNull(authority) || authority.getAccess() == 0) { - return Result.buildFrom(ResultStatus.USER_WITHOUT_AUTHORITY); - } - if (authority.getAccess() == 1) { - //可以消费 - dto.setProduceQuota(null); - } - if (authority.getAccess() == 2) { - //可以生产 - dto.setConsumeQuota(null); - } - //构建topicquota - TopicQuota topicQuotaDO = new TopicQuota(); - topicQuotaDO.setAppId(dto.getAppId()); - topicQuotaDO.setClusterId(physicalClusterId); - topicQuotaDO.setTopicName(dto.getTopicName()); - topicQuotaDO.setConsumeQuota(dto.getConsumeQuota()); - topicQuotaDO.setProduceQuota(dto.getProduceQuota()); - //添加配额 - if (quotaService.addTopicQuota(topicQuotaDO) > 0) { + dto.setClusterId(physicalClusterId); + // 添加配额 + if (quotaService.addTopicQuota(TopicQuota.buildFrom(dto)) > 0) { return Result.buildFrom(ResultStatus.SUCCESS); } return Result.buildFrom(ResultStatus.MYSQL_ERROR); } - @ApiOperation(value = "权限调整",notes = "权限调整") - @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) - @ResponseBody - public Result addAuthority(@RequestBody TopicAuthorityDTO dto) { - //非空校验 - if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - //查询该用户拥有的应用 - List appDOs = appService.getByPrincipal(SpringTool.getUserName()); - if (ValidateUtils.isEmptyList(appDOs)) { - //该用户无应用，需要先申请应用 - return Result.buildFrom(ResultStatus.APP_NOT_EXIST); - } - List appIds = appDOs.stream().map(AppDO::getId).collect(Collectors.toList()); - if (!appIds.contains(dto.getAccess())) { - //入参中的appId，该用户未拥有 - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - //获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - //集群不存在 - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - //获取集群信息 - ClusterDO clusterDO = clusterService.getById(physicalClusterId); - if (ValidateUtils.isNull(clusterDO)) { - //集群不存在 - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - TopicDO topic = topicManagerService.getByTopicName(physicalClusterId, dto.getTopicName()); - if (ValidateUtils.isNull(topic)) { - //topic不存在 - return Result.buildFrom(ResultStatus.TOPIC_NOT_EXIST); - } - //构建authorityDo - AuthorityDO authorityDO = new AuthorityDO(); - authorityDO.setClusterId(physicalClusterId); - authorityDO.setAppId(dto.getAppId()); - authorityDO.setTopicName(dto.getTopicName()); - authorityDO.setAccess(dto.getAccess()); - if (authorityService.addAuthority(authorityDO) > 0) { - return Result.buildFrom(ResultStatus.SUCCESS); - } - return Result.buildFrom(ResultStatus.MYSQL_ERROR); - } } From c3a0dbbe485eb23a1f764c5a289461c100b917ea Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 18 May 2021 11:45:15 +0800 Subject: [PATCH 26/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E7=BB=95=E8=BF=87=E7=9A=84=E6=8E=A5=E5=8F=A3=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/call_api_bypass_login.md | 49 ++++++++++++++++++ docs/user_guide/faq.md | 5 ++ .../common/constant/TrickLoginConstant.java | 21 ++++++++ .../manager/common/utils/SpringTool.java | 24 +++++---- .../kafka/manager/account/LoginService.java | 20 ++++++++ .../component/AbstractSingleSignOn.java | 2 +- .../login/trick/TrickLoginService.java | 50 +++++++++++++++++++ .../account/impl/LoginServiceImpl.java | 6 ++- 8 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 docs/user_guide/call_api_bypass_login.md create mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java create mode 100644 kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/login/trick/TrickLoginService.java diff --git a/docs/user_guide/call_api_bypass_login.md b/docs/user_guide/call_api_bypass_login.md new file mode 100644 index 00000000..7a2feac8 --- /dev/null +++ b/docs/user_guide/call_api_bypass_login.md @@ -0,0 +1,49 @@ + +--- + +![kafka-manager-logo](../assets/images/common/logo_name.png) + +**一站式`Apache Kafka`集群指标监控与运维管控平台** + +--- + +# 登录绕过 + +## 背景 + +现在除了开放出来的第三方接口，其他接口都需要走登录认证。 + +但是第三方接口不多，开放出来的能力有限，但是登录的接口又需要登录，非常的麻烦。 + +因此，新增了一个登录绕过的功能，为一些紧急临时的需求，提供一个调用不需要登录的能力。 + +## 使用方式 + +步骤一：接口调用时，在header中，增加如下信息： +```shell +# 表示开启登录绕过 +Trick-Login-Switch : on + +# 登录绕过的用户, 这里可以是admin, 或者是其他的, 但是必须在运维管控->平台管理->用户管理中设置了该用户。 +Trick-Login-User : admin +``` + +  + +步骤二：在运维管控->平台管理->平台配置上，设置允许了该用户以绕过的方式登录 +```shell +# 设置的key，必须是这个 +SECURITY.TRICK_USERS + +# 设置的value，是json数组的格式，例如 +[ "admin", "logi"] +``` + +  + +步骤三：解释说明 + +设置完成上面两步之后，就可以直接调用需要登录的接口了。 + +但是还有一点需要注意，绕过的用户仅能调用他有权限的接口，比如一个普通用户，那么他就只能调用普通的接口，不能去调用运维人员的接口。 + diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 54ac5543..7fec9050 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -27,6 +27,7 @@ - 15、APP(应用)如何被使用起来？ - 16、为什么下线应用提示operation forbidden？ - 17、删除Topic成功，为什么过一会儿之后又出现了？ +- 18、如何在不登录的情况下，调用一些需要登录的接口？ --- @@ -195,3 +196,7 @@ for (int i= 0; i < 100000; ++i) { producer.send(new ProducerRecord("logi_km" + i,"hello logi_km")); } ``` + +### 18、如何在不登录的情况下，调用一些需要登录的接口？ + +具体见：[登录绕过](./call_api_bypass_login.md) diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java new file mode 100644 index 00000000..159e1c61 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java @@ -0,0 +1,21 @@ +package com.xiaojukeji.kafka.manager.common.constant; + +public class TrickLoginConstant { + /** + * HTTP Header key + */ + public static final String TRICK_LOGIN_SWITCH = "Trick-Login-Switch"; + + public static final String TRICK_LOGIN_USER = "Trick-Login-User"; + + /** + * 配置允许 trick 登录用户名单 + */ + public static final String TRICK_LOGIN_LEGAL_USER_CONFIG_KEY = "SECURITY.TRICK_USERS"; + + /** + * 开关状态值 + */ + public static final String TRICK_LOGIN_SWITCH_ON = "on"; + public static final String TRICK_LOGIN_SWITCH_OFF = "off"; +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/SpringTool.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/SpringTool.java index de0783d2..50723ebf 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/SpringTool.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/SpringTool.java @@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.common.utils; import com.xiaojukeji.kafka.manager.common.constant.Constant; import com.xiaojukeji.kafka.manager.common.constant.LoginConstant; +import com.xiaojukeji.kafka.manager.common.constant.TrickLoginConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; @@ -53,13 +54,6 @@ public class SpringTool implements ApplicationContextAware, DisposableBean { return getApplicationContext().getBeansOfType(type); } -// /** -// * 从静态变量applicationContext中去的Bean，自动转型为所复制对象的类型 -// */ -// public static T getBean(Class requiredType) { -// return (T) applicationContext.getBean(requiredType); -// } - /** * 清除SpringContextHolder中的ApplicationContext为Null */ @@ -87,10 +81,18 @@ public class SpringTool implements ApplicationContextAware, DisposableBean { } public static String getUserName(){ - HttpServletRequest request = - ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest(); - HttpSession session = request.getSession(); - String username = (String) session.getAttribute(LoginConstant.SESSION_USERNAME_KEY); + HttpServletRequest request = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest(); + + String username = null; + if (TrickLoginConstant.TRICK_LOGIN_SWITCH_ON.equals(request.getHeader(TrickLoginConstant.TRICK_LOGIN_SWITCH))) { + // trick登录方式的获取用户 + username = request.getHeader(TrickLoginConstant.TRICK_LOGIN_USER); + } else { + // 走页面登录方式登录的获取用户 + HttpSession session = request.getSession(); + username = (String) session.getAttribute(LoginConstant.SESSION_USERNAME_KEY); + } + if (ValidateUtils.isNull(username)) { return Constant.DEFAULT_USER_NAME; } diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/LoginService.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/LoginService.java index 707d4908..48d3f710 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/LoginService.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/LoginService.java @@ -12,9 +12,29 @@ import javax.servlet.http.HttpServletResponse; * @date 20/8/20 */ public interface LoginService { + /** + * 登录 + * @param request HttpServletRequest + * @param response HttpServletResponse + * @param dto 登录信息 + * @return 登录结果 + */ Result login(HttpServletRequest request, HttpServletResponse response, LoginDTO dto); + /** + * 登出 + * @param request HttpServletRequest + * @param response HttpServletResponse + * @param needJump2LoginPage 是否需要跳转到登录页 + */ void logout(HttpServletRequest request, HttpServletResponse response, Boolean needJump2LoginPage); + /** + * 检查是否登录 + * @param request HttpServletRequest + * @param response HttpServletResponse + * @param classRequestMappingValue request-mapping的value + * @return 检查结果, false:未登录或无权限, true:已登录并且有权限 + */ boolean checkLogin(HttpServletRequest request, HttpServletResponse response, String classRequestMappingValue); } \ No newline at end of file diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/AbstractSingleSignOn.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/AbstractSingleSignOn.java index d6257364..4a67f8f4 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/AbstractSingleSignOn.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/AbstractSingleSignOn.java @@ -7,7 +7,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** - * 单点登录抽象类 + * 登录抽象类 * @author zengqiao * @date 20/8/20 */ diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/login/trick/TrickLoginService.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/login/trick/TrickLoginService.java new file mode 100644 index 00000000..08195664 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/login/trick/TrickLoginService.java @@ -0,0 +1,50 @@ +package com.xiaojukeji.kafka.manager.account.component.login.trick; + +import com.xiaojukeji.kafka.manager.common.constant.TrickLoginConstant; +import com.xiaojukeji.kafka.manager.service.service.ConfigService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; +import java.util.Optional; + + +/** + * @author zengqiao + * @date 21/5/18 + */ +@Service +public class TrickLoginService { + private final static Logger LOGGER = LoggerFactory.getLogger(TrickLoginService.class); + + @Autowired + private ConfigService configService; + + /** + * 是否开启trick的方式登录 + */ + public boolean isTrickLoginOn(HttpServletRequest request) { + return TrickLoginConstant.TRICK_LOGIN_SWITCH_ON.equals(request.getHeader(TrickLoginConstant.TRICK_LOGIN_SWITCH)); + } + + /** + * 开启trick方式登录后，当前用户是否可以登录 + */ + public String checkTrickLogin(HttpServletRequest request) { + String trickLoginUser = request.getHeader(TrickLoginConstant.TRICK_LOGIN_USER); + LOGGER.info("class=TrickLoginService||method=checkTrickLogin||user={}||uri={}||msg=try trick login", trickLoginUser, request.getRequestURI()); + if (!checkTrickLogin(trickLoginUser)) { + LOGGER.warn("class=TrickLoginService||method=checkTrickLogin||user={}||uri={}||msg=trick login failed", trickLoginUser, request.getRequestURI()); + return null; + } + return trickLoginUser; + } + + private boolean checkTrickLogin(String trickLoginUser) { + return Optional.ofNullable(configService.getArrayByKey(TrickLoginConstant.TRICK_LOGIN_LEGAL_USER_CONFIG_KEY, String.class)) + .filter(names -> names.contains(trickLoginUser)) + .isPresent(); + } +} diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/LoginServiceImpl.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/LoginServiceImpl.java index 7a803603..8f079fde 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/LoginServiceImpl.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/LoginServiceImpl.java @@ -3,6 +3,7 @@ package com.xiaojukeji.kafka.manager.account.impl; import com.xiaojukeji.kafka.manager.account.AccountService; import com.xiaojukeji.kafka.manager.account.component.AbstractSingleSignOn; import com.xiaojukeji.kafka.manager.account.LoginService; +import com.xiaojukeji.kafka.manager.account.component.login.trick.TrickLoginService; import com.xiaojukeji.kafka.manager.common.bizenum.AccountRoleEnum; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.common.constant.LoginConstant; @@ -31,6 +32,9 @@ public class LoginServiceImpl implements LoginService { @Autowired private AccountService accountService; + @Autowired + private TrickLoginService trickLoginService; + @Autowired private AbstractSingleSignOn singleSignOn; @@ -80,7 +84,7 @@ public class LoginServiceImpl implements LoginService { return true; } - String username = singleSignOn.checkLoginAndGetLdap(request); + String username = trickLoginService.isTrickLoginOn(request)? trickLoginService.checkTrickLogin(request): singleSignOn.checkLoginAndGetLdap(request); if (ValidateUtils.isBlank(username)) { // 未登录, 则返回false, 同时重定向到登录页面 singleSignOn.setRedirectToLoginPage(response); From c5b1bed7dcddb071830bad1a3fe794dd7f1b6e58 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 18 May 2021 14:13:23 +0800 Subject: [PATCH 27/64] =?UTF-8?q?=E7=AE=80=E5=8C=96sd=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=9E=9A=E4=B8=BE=E7=B1=BB=E7=9A=84=E9=95=BF=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/create_mysql_table.sql | 10 +++++----- .../common/bizenum/gateway/GatewayConfigKeyEnum.java | 10 +++++----- .../service/gateway/impl/GatewayConfigServiceImpl.java | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/install_guide/create_mysql_table.sql b/docs/install_guide/create_mysql_table.sql index 065532eb..12910ae1 100644 --- a/docs/install_guide/create_mysql_table.sql +++ b/docs/install_guide/create_mysql_table.sql @@ -210,11 +210,11 @@ CREATE TABLE `gateway_config` ( PRIMARY KEY (`id`), UNIQUE KEY `uniq_type_name` (`type`,`name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='gateway配置'; -INSERT INTO gateway_config(type, name, value, `version`) values('SERVICE_DISCOVERY_QUEUE_SIZE', 'SERVICE_DISCOVERY_QUEUE_SIZE', 100000000, 1); -INSERT INTO gateway_config(type, name, value, `version`) values('SERVICE_DISCOVERY_APPID_RATE', 'SERVICE_DISCOVERY_APPID_RATE', 100000000, 1); -INSERT INTO gateway_config(type, name, value, `version`) values('SERVICE_DISCOVERY_IP_RATE', 'SERVICE_DISCOVERY_IP_RATE', 100000000, 1); -INSERT INTO gateway_config(type, name, value, `version`) values('SERVICE_DISCOVERY_SP_RATE', 'app_01234567', 100000000, 1); -INSERT INTO gateway_config(type, name, value, `version`) values('SERVICE_DISCOVERY_SP_RATE', '192.168.0.1', 100000000, 1); +INSERT INTO gateway_config(type, name, value, `version`, `description`) values('SD_QUEUE_SIZE', 'SD_QUEUE_SIZE', 100000000, 1, '任意集群队列大小'); +INSERT INTO gateway_config(type, name, value, `version`, `description`) values('SD_APP_RATE', 'SD_APP_RATE', 100000000, 1, '任意一个App限速'); +INSERT INTO gateway_config(type, name, value, `version`, `description`) values('SD_IP_RATE', 'SD_IP_RATE', 100000000, 1, '任意一个IP限速'); +INSERT INTO gateway_config(type, name, value, `version`, `description`) values('SD_SP_RATE', 'app_01234567', 100000000, 1, '指定App限速'); +INSERT INTO gateway_config(type, name, value, `version`, `description`) values('SD_SP_RATE', '192.168.0.1', 100000000, 1, '指定IP限速'); -- -- Table structure for table `heartbeat` diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java index 226cc5c6..5562138f 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java @@ -5,11 +5,11 @@ package com.xiaojukeji.kafka.manager.common.bizenum.gateway; * @date 20/7/28 */ public enum GatewayConfigKeyEnum { - SD_CLUSTER_ID("SERVICE_DISCOVERY_CLUSTER_ID", "SERVICE_DISCOVERY_CLUSTER_ID"), - SD_QUEUE_SIZE("SERVICE_DISCOVERY_QUEUE_SIZE", "SERVICE_DISCOVERY_QUEUE_SIZE"), - SD_APP_ID_RATE("SERVICE_DISCOVERY_APPID_RATE", "SERVICE_DISCOVERY_APPID_RATE"), - SD_IP_RATE("SERVICE_DISCOVERY_IP_RATE", "SERVICE_DISCOVERY_IP_RATE"), - SD_SP_RATE("SERVICE_DISCOVERY_SP_RATE", "SERVICE_DISCOVERY_SP_RATE"), + SD_CLUSTER_ID("SD_CLUSTER_ID", "SD_CLUSTER_ID"), + SD_QUEUE_SIZE("SD_QUEUE_SIZE", "SD_QUEUE_SIZE"), + SD_APP_RATE("SD_APP_RATE", "SD_APP_RATE"), + SD_IP_RATE("SD_IP_RATE", "SD_IP_RATE"), + SD_SP_RATE("SD_SP_RATE", "SD_SP_RATE"), ; diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java index 18ee0a0d..0ceb3b30 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java @@ -86,8 +86,8 @@ public class GatewayConfigServiceImpl implements GatewayConfigService { GatewayConfigDO configDO = null; try { configDO = gatewayConfigDao.getByConfigTypeAndName( - GatewayConfigKeyEnum.SD_APP_ID_RATE.getConfigType(), - GatewayConfigKeyEnum.SD_APP_ID_RATE.getConfigName() + GatewayConfigKeyEnum.SD_APP_RATE.getConfigType(), + GatewayConfigKeyEnum.SD_APP_RATE.getConfigName() ); if (ValidateUtils.isNull(configDO) || configDO.getVersion() <= requestVersion) { return new AppRateConfig(Long.MIN_VALUE, null); From 892e195f0e10e26786dd49ff0184dc22aef60f49 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 18 May 2021 14:33:02 +0800 Subject: [PATCH 28/64] code format --- .../manager/common/bizenum/ApiLevelEnum.java | 19 ------------------- .../common/bizenum/ConsumeHealthEnum.java | 8 -------- .../manager/common/bizenum/DBStatusEnum.java | 7 +++++-- .../kafka/manager/common/bizenum/IDCEnum.java | 8 -------- .../common/bizenum/KafkaBrokerRoleEnum.java | 4 ---- .../common/bizenum/KafkaClientEnum.java | 8 -------- .../common/bizenum/OffsetResetTypeEnum.java | 7 +++++++ .../common/bizenum/OperationStatusEnum.java | 8 ++++++++ .../common/bizenum/PeakFlowStatusEnum.java | 12 ++---------- .../bizenum/RebalanceDimensionEnum.java | 8 ++++++++ .../common/bizenum/TaskStatusEnum.java | 13 +------------ .../bizenum/TaskStatusReassignEnum.java | 7 +------ .../common/bizenum/TopicAuthorityEnum.java | 8 ++++++++ .../bizenum/TopicExpiredStatusEnum.java | 8 ++++++++ .../bizenum/TopicOffsetChangedEnum.java | 8 -------- .../bizenum/gateway/GatewayConfigKeyEnum.java | 8 -------- .../common/constant/ApiLevelContent.java | 3 +++ .../manager/common/constant/ApiPrefix.java | 3 +++ .../common/constant/ConfigConstant.java | 3 +++ .../manager/common/constant/Constant.java | 3 +++ .../common/constant/KafkaConstant.java | 3 +++ .../constant/KafkaMetricsCollections.java | 3 +++ .../manager/common/constant/LogConstant.java | 3 +++ .../common/constant/LoginConstant.java | 3 +++ .../common/constant/SystemCodeConstant.java | 3 +++ .../constant/TopicCreationConstant.java | 3 +++ .../common/constant/TopicSampleConstant.java | 3 +++ .../common/constant/TrickLoginConstant.java | 3 +++ .../manager/common/zookeeper/ZkPathUtil.java | 3 +++ .../service/impl/BrokerServiceImpl.java | 2 +- 30 files changed, 88 insertions(+), 94 deletions(-) delete mode 100644 kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ApiLevelEnum.java diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ApiLevelEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ApiLevelEnum.java deleted file mode 100644 index 73be0d16..00000000 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ApiLevelEnum.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.xiaojukeji.kafka.manager.common.bizenum; - -/** - * @author zengqiao - * @date 20/7/27 - */ -public enum ApiLevelEnum { - LEVEL_0(0), - LEVEL_1(1), - LEVEL_2(2), - LEVEL_3(3) - ; - - private int level; - - ApiLevelEnum(int level) { - this.level = level; - } -} \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ConsumeHealthEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ConsumeHealthEnum.java index 74d2d8ab..f5cda2ed 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ConsumeHealthEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ConsumeHealthEnum.java @@ -24,18 +24,10 @@ public enum ConsumeHealthEnum { return code; } - public void setCode(Integer code) { - this.code = code; - } - public String getMessage() { return message; } - public void setMessage(String message) { - this.message = message; - } - @Override public String toString() { return "ConsumeHealthEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/DBStatusEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/DBStatusEnum.java index 4f6fb1cf..89518f83 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/DBStatusEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/DBStatusEnum.java @@ -19,7 +19,10 @@ public enum DBStatusEnum { return status; } - public void setStatus(int status) { - this.status = status; + @Override + public String toString() { + return "DBStatusEnum{" + + "status=" + status + + '}'; } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java index 73569d56..2b3cad7c 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java @@ -21,18 +21,10 @@ public enum IDCEnum { return idc; } - public void setIdc(String idc) { - this.idc = idc; - } - public String getName() { return name; } - public void setName(String name) { - this.name = name; - } - @Override public String toString() { return "IDCEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaBrokerRoleEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaBrokerRoleEnum.java index befd5257..246b4b5e 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaBrokerRoleEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaBrokerRoleEnum.java @@ -21,10 +21,6 @@ public enum KafkaBrokerRoleEnum { return role; } - public void setRole(String role) { - this.role = role; - } - @Override public String toString() { return "KafkaBrokerRoleEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaClientEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaClientEnum.java index 6e5bea6f..0b35277e 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaClientEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/KafkaClientEnum.java @@ -24,18 +24,10 @@ public enum KafkaClientEnum { return code; } - public void setCode(Integer code) { - this.code = code; - } - public String getName() { return name; } - public void setName(String name) { - this.name = name; - } - @Override public String toString() { return "KafkaClientEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OffsetResetTypeEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OffsetResetTypeEnum.java index b69a8a25..170946e8 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OffsetResetTypeEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OffsetResetTypeEnum.java @@ -18,4 +18,11 @@ public enum OffsetResetTypeEnum { public Integer getCode() { return code; } + + @Override + public String toString() { + return "OffsetResetTypeEnum{" + + "code=" + code + + '}'; + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OperationStatusEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OperationStatusEnum.java index cf8f53d2..b88135a5 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OperationStatusEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/OperationStatusEnum.java @@ -27,4 +27,12 @@ public enum OperationStatusEnum { public String getMessage() { return message; } + + @Override + public String toString() { + return "OperationStatusEnum{" + + "code=" + code + + ", message='" + message + '\'' + + '}'; + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/PeakFlowStatusEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/PeakFlowStatusEnum.java index 9b71f038..f39ac91a 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/PeakFlowStatusEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/PeakFlowStatusEnum.java @@ -15,9 +15,9 @@ public enum PeakFlowStatusEnum { ; - public Integer code; + private Integer code; - public String message; + private String message; PeakFlowStatusEnum(Integer code, String message) { this.code = code; @@ -28,18 +28,10 @@ public enum PeakFlowStatusEnum { return code; } - public void setCode(Integer code) { - this.code = code; - } - public String getMessage() { return message; } - public void setMessage(String message) { - this.message = message; - } - @Override public String toString() { return "PeakFlowStatusEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java index e196e8c2..c5259461 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java @@ -29,4 +29,12 @@ public enum RebalanceDimensionEnum { public String getMessage() { return message; } + + @Override + public String toString() { + return "RebalanceDimensionEnum{" + + "code=" + code + + ", message='" + message + '\'' + + '}'; + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusEnum.java index ebf3dc82..a478eafe 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusEnum.java @@ -43,18 +43,10 @@ public enum TaskStatusEnum { return code; } - public void setCode(Integer code) { - this.code = code; - } - public String getMessage() { return message; } - public void setMessage(String message) { - this.message = message; - } - @Override public String toString() { return "TaskStatusEnum{" + @@ -64,9 +56,6 @@ public enum TaskStatusEnum { } public static Boolean isFinished(Integer code) { - if (code >= FINISHED.getCode()) { - return true; - } - return false; + return code >= FINISHED.getCode(); } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusReassignEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusReassignEnum.java index 7fd6ef8f..fc8adcc1 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusReassignEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TaskStatusReassignEnum.java @@ -45,11 +45,6 @@ public enum TaskStatusReassignEnum { } public static Boolean isFinished(Integer code) { - if (SUCCEED.getCode().equals(code) - || FAILED.getCode().equals(code) - || CANCELED.getCode().equals(code)) { - return true; - } - return false; + return SUCCEED.getCode().equals(code) || FAILED.getCode().equals(code) || CANCELED.getCode().equals(code); } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicAuthorityEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicAuthorityEnum.java index 2cfba027..7abafb8c 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicAuthorityEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicAuthorityEnum.java @@ -33,4 +33,12 @@ public enum TopicAuthorityEnum { public String getMessage() { return message; } + + @Override + public String toString() { + return "TopicAuthorityEnum{" + + "code=" + code + + ", message='" + message + '\'' + + '}'; + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java index bac44235..6a2f32c1 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicExpiredStatusEnum.java @@ -29,4 +29,12 @@ public enum TopicExpiredStatusEnum { public String getMessage() { return message; } + + @Override + public String toString() { + return "TopicExpiredStatusEnum{" + + "status=" + status + + ", message='" + message + '\'' + + '}'; + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicOffsetChangedEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicOffsetChangedEnum.java index ecb6b2f1..4c88f25c 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicOffsetChangedEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/TopicOffsetChangedEnum.java @@ -23,18 +23,10 @@ public enum TopicOffsetChangedEnum { return code; } - public void setCode(Integer code) { - this.code = code; - } - public String getMessage() { return message; } - public void setMessage(String message) { - this.message = message; - } - @Override public String toString() { return "TopicOffsetChangedEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java index 5562138f..b3403e69 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java @@ -26,18 +26,10 @@ public enum GatewayConfigKeyEnum { return configType; } - public void setConfigType(String configType) { - this.configType = configType; - } - public String getConfigName() { return configName; } - public void setConfigName(String configName) { - this.configName = configName; - } - @Override public String toString() { return "GatewayConfigKeyEnum{" + diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiLevelContent.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiLevelContent.java index 2447564f..8136cd16 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiLevelContent.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiLevelContent.java @@ -12,4 +12,7 @@ public class ApiLevelContent { public static final int LEVEL_NORMAL_3 = 3; public static final int LEVEL_DEFAULT_4 = 4; + + private ApiLevelContent() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiPrefix.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiPrefix.java index 1aec18f2..b0f84405 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiPrefix.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ApiPrefix.java @@ -28,4 +28,7 @@ public class ApiPrefix { // gateway public static final String GATEWAY_API_V1_PREFIX = "/gateway" + API_V1_PREFIX; + + private ApiPrefix() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ConfigConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ConfigConstant.java index faca17b0..361c841f 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ConfigConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ConfigConstant.java @@ -30,4 +30,7 @@ public class ConfigConstant { public static final String BROKER_CAPACITY_LIMIT_CONFIG_KEY = "BROKER_CAPACITY_LIMIT_CONFIG"; public static final String KAFKA_CLUSTER_DO_CONFIG_KEY = "KAFKA_CLUSTER_DO_CONFIG"; + + private ConfigConstant() { + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/Constant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/Constant.java index 81c1dc89..7ecc295b 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/Constant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/Constant.java @@ -45,4 +45,7 @@ public class Constant { public static final Integer DEFAULT_MAX_CAL_TOPIC_EXPIRED_DAY = 90; public static final Integer INVALID_CODE = -1; + + private Constant() { + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java index 92425303..4d69f914 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java @@ -16,4 +16,7 @@ public class KafkaConstant { public static final String CLIENT_VERSION_NAME_UNKNOWN = "unknown"; public static final String RETENTION_MS_KEY = "retention.ms"; + + private KafkaConstant() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaMetricsCollections.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaMetricsCollections.java index 9c7f59e3..be82317a 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaMetricsCollections.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaMetricsCollections.java @@ -39,4 +39,7 @@ public class KafkaMetricsCollections { * Broker信息 */ public static final int BROKER_VERSION = 400; + + private KafkaMetricsCollections() { + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LogConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LogConstant.java index 55fa756f..bba7670c 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LogConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LogConstant.java @@ -10,4 +10,7 @@ public class LogConstant { public static final String API_METRICS_LOGGER = "API_METRICS_LOGGER"; public static final String SCHEDULED_TASK_LOGGER = "SCHEDULED_TASK_LOGGER"; + + private LogConstant() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LoginConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LoginConstant.java index bc95dbc2..8c9b47aa 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LoginConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/LoginConstant.java @@ -11,4 +11,7 @@ public class LoginConstant { public static final String COOKIE_CHINESE_USERNAME_KEY = "chineseName"; public static final Integer COOKIE_OR_SESSION_MAX_AGE_UNIT_MS = 24 * 60 * 60 * 1000; + + private LoginConstant() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/SystemCodeConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/SystemCodeConstant.java index c3162a4b..510a90c1 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/SystemCodeConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/SystemCodeConstant.java @@ -6,4 +6,7 @@ package com.xiaojukeji.kafka.manager.common.constant; */ public class SystemCodeConstant { public static final String KAFKA_MANAGER = "kafka-manager"; + + private SystemCodeConstant() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java index b8c361b3..3a6dd478 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java @@ -54,4 +54,7 @@ public class TopicCreationConstant { * 单次自动化审批, 最多允许的通过单子 */ public static final Integer MAX_PASSED_ORDER_NUM_PER_TASK = 200; + + private TopicCreationConstant() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicSampleConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicSampleConstant.java index 5ee15331..d409e862 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicSampleConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicSampleConstant.java @@ -16,4 +16,7 @@ public class TopicSampleConstant { public static final Integer MAX_TIMEOUT_UNIT_MS = 10000; public static final Integer POLL_TIME_OUT_UNIT_MS = 2000; public static final Integer MAX_DATA_LENGTH_UNIT_BYTE = 2048; + + private TopicSampleConstant() { + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java index 159e1c61..0bb92d2e 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TrickLoginConstant.java @@ -18,4 +18,7 @@ public class TrickLoginConstant { */ public static final String TRICK_LOGIN_SWITCH_ON = "on"; public static final String TRICK_LOGIN_SWITCH_OFF = "off"; + + private TrickLoginConstant() { + } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java index 6705f435..0410a553 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java @@ -119,4 +119,7 @@ public class ZkPathUtil { public static String getControllerCandidatePath(Integer brokerId) { return D_CONTROLLER_CANDIDATES + ZOOKEEPER_SEPARATOR + brokerId; } + + private ZkPathUtil() { + } } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/BrokerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/BrokerServiceImpl.java index 12af2e18..24eea55f 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/BrokerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/BrokerServiceImpl.java @@ -91,7 +91,7 @@ public class BrokerServiceImpl implements BrokerService { for (BrokerDO brokerDO : brokerDOList) { PeakFlowStatusEnum peakFlowStatus = getPeakFlowStatus(brokerDO.getMaxAvgBytesIn(), peakFlow); peakFlowStatusMap.put( - peakFlowStatus.code, + peakFlowStatus.getCode(), peakFlowStatusMap.getOrDefault(peakFlowStatus.getCode(), 0) + 1 ); } From d4ee5e91a20a6d98068707f2bcc2bc711306d11d Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 18 May 2021 14:49:00 +0800 Subject: [PATCH 29/64] =?UTF-8?q?=E9=85=8D=E9=A2=9D=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/service/TopicManagerService.java | 8 +++++ .../service/impl/TopicManagerServiceImpl.java | 35 +++++++++++++++++++ .../thirdpart/ThirdPartTopicController.java | 22 ++---------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java index 8dc0e0c1..a19d8220 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java @@ -3,6 +3,7 @@ package com.xiaojukeji.kafka.manager.service.service; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.RdTopicBasic; +import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicAppData; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicBusinessInfo; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicDTO; @@ -122,5 +123,12 @@ public interface TopicManagerService { List getTopicStatistic(Long clusterId, String topicName, Date startTime, Date endTime); TopicBusinessInfo getTopicBusinessInfo(Long physicalClusterId, String topicName); + + /** + * topic配额调整 + * @param topicQuota topic配额 + * @return + */ + ResultStatus addTopicQuota(TopicQuota topicQuota); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java index bce5fbe7..36f6f593 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java @@ -10,6 +10,7 @@ import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.RdTopicBasic; +import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.MineTopicSummary; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicAppData; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicBusinessInfo; @@ -34,6 +35,7 @@ import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; +import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,6 +88,9 @@ public class TopicManagerServiceImpl implements TopicManagerService { @Autowired private OperateRecordService operateRecordService; + @Autowired + private QuotaService quotaService; + @Override public List listAll() { try { @@ -618,6 +623,36 @@ public class TopicManagerServiceImpl implements TopicManagerService { return topicBusinessInfo; } + @Override + public ResultStatus addTopicQuota(TopicQuota topicQuota) { + // 获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(topicQuota.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + return ResultStatus.CLUSTER_NOT_EXIST; + } + // 权限判断(access 0:无权限, 1:读, 2:写, 3:读写，4:可管理) + AuthorityDO authority = authorityService.getAuthority(physicalClusterId, + topicQuota.getTopicName(), topicQuota.getAppId()); + if (ValidateUtils.isNull(authority) || authority.getAccess() == TopicAuthorityEnum.DENY.getCode()) { + return ResultStatus.USER_WITHOUT_AUTHORITY; + } + if (authority.getAccess() == TopicAuthorityEnum.READ.getCode()) { + // 可以消费 + topicQuota.setProduceQuota(null); + } + if (authority.getAccess() == TopicAuthorityEnum.WRITE.getCode()) { + // 可以生产 + topicQuota.setConsumeQuota(null); + } + // 设置物理集群id + topicQuota.setClusterId(physicalClusterId); + // 添加配额 + if (quotaService.addTopicQuota(topicQuota) > 0) { + return ResultStatus.SUCCESS; + } + return ResultStatus.MYSQL_ERROR; + } + private RdTopicBasic convert2RdTopicBasic(ClusterDO clusterDO, String topicName, TopicDO topicDO, diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index a1940e4c..762ed3a9 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -17,11 +17,9 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; -import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; -import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter; import com.xiaojukeji.kafka.manager.web.converters.TopicModelConverter; @@ -56,12 +54,6 @@ public class ThirdPartTopicController { @Autowired private TopicManagerService topicManagerService; - @Autowired - private QuotaService quotaService; - - @Autowired - private LogicalClusterMetadataManager logicalClusterMetadataManager; - @ApiOperation(value = "Topic元信息", notes = "LogX调用") @RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET) @ResponseBody @@ -148,21 +140,11 @@ public class ThirdPartTopicController { @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) @ResponseBody public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { - //非空校验 + // 非空校验 if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } - //获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(dto.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - dto.setClusterId(physicalClusterId); - // 添加配额 - if (quotaService.addTopicQuota(TopicQuota.buildFrom(dto)) > 0) { - return Result.buildFrom(ResultStatus.SUCCESS); - } - return Result.buildFrom(ResultStatus.MYSQL_ERROR); + return Result.buildFrom(topicManagerService.addTopicQuota(TopicQuota.buildFrom(dto))); } } From 52244325d91b1d975cedf965bc22a84a1cda37e8 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 18 May 2021 16:03:24 +0800 Subject: [PATCH 30/64] =?UTF-8?q?=E6=9D=83=E9=99=90=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/service/TopicManagerService.java | 8 +++++ .../service/impl/TopicManagerServiceImpl.java | 33 +++++++++++++++++++ .../openapi/common/dto/TopicAuthorityDTO.java | 2 +- .../thirdpart/ThirdPartTopicController.java | 12 +++++++ .../web/converters/AuthorityConverter.java | 15 +++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/AuthorityConverter.java diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java index a19d8220..5e7e7a75 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java @@ -11,6 +11,7 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.topic.MineTopicSummary; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicExpiredDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicStatisticsDO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import java.util.Date; import java.util.List; @@ -130,5 +131,12 @@ public interface TopicManagerService { * @return */ ResultStatus addTopicQuota(TopicQuota topicQuota); + + /** + * topic权限调整 + * @param authorityDO topic权限 + * @return + */ + ResultStatus addAuthority(AuthorityDO authorityDO); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java index 36f6f593..a5d459b4 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java @@ -21,6 +21,7 @@ import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.utils.DateUtils; import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; import com.xiaojukeji.kafka.manager.common.utils.NumberUtils; +import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.config.TopicQuotaData; @@ -653,6 +654,38 @@ public class TopicManagerServiceImpl implements TopicManagerService { return ResultStatus.MYSQL_ERROR; } + @Override + public ResultStatus addAuthority(AuthorityDO authorityDO) { + // 查询该用户拥有的应用 + List appDOs = appService.getByPrincipal(SpringTool.getUserName()); + if (ValidateUtils.isEmptyList(appDOs)) { + // 该用户无应用，需要先申请应用 + return ResultStatus.APP_NOT_EXIST; + } + List appIds = appDOs.stream().map(AppDO::getId).collect(Collectors.toList()); + if (!appIds.contains(authorityDO.getAppId())) { + // 入参中的appId，该用户未拥有 + return ResultStatus.APP_NOT_EXIST; + } + // 获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(authorityDO.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + // 集群不存在 + return ResultStatus.CLUSTER_NOT_EXIST; + } + TopicDO topic = getByTopicName(physicalClusterId, authorityDO.getTopicName()); + if (ValidateUtils.isNull(topic)) { + // topic不存在 + return ResultStatus.TOPIC_NOT_EXIST; + } + // 设置物理集群id + authorityDO.setClusterId(physicalClusterId); + if (authorityService.addAuthority(authorityDO) > 0) { + return ResultStatus.SUCCESS; + } + return ResultStatus.MYSQL_ERROR; + } + private RdTopicBasic convert2RdTopicBasic(ClusterDO clusterDO, String topicName, TopicDO topicDO, diff --git a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java index 564b31d6..b5394f6b 100644 --- a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java +++ b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java @@ -11,7 +11,7 @@ public class TopicAuthorityDTO extends ClusterTopicDTO { @ApiModelProperty(value = "appId") private String appId; - @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写") + @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写, 4:可管理") private Integer access; public String getAppId() { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index 762ed3a9..93907371 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -14,12 +14,14 @@ import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGro import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicAuthorizedAppVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicRequestTimeDetailVO; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; +import com.xiaojukeji.kafka.manager.openapi.common.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; +import com.xiaojukeji.kafka.manager.web.converters.AuthorityConverter; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter; import com.xiaojukeji.kafka.manager.web.converters.TopicModelConverter; @@ -147,4 +149,14 @@ public class ThirdPartTopicController { return Result.buildFrom(topicManagerService.addTopicQuota(TopicQuota.buildFrom(dto))); } + @ApiOperation(value = "权限调整",notes = "权限调整") + @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) + @ResponseBody + public Result addAuthority(@RequestBody TopicAuthorityDTO dto) { + //非空校验 + if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return Result.buildFrom(topicManagerService.addAuthority(AuthorityConverter.convert2AuthorityDO(dto))); + } } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/AuthorityConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/AuthorityConverter.java new file mode 100644 index 00000000..227a16a2 --- /dev/null +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/AuthorityConverter.java @@ -0,0 +1,15 @@ +package com.xiaojukeji.kafka.manager.web.converters; + +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; +import com.xiaojukeji.kafka.manager.openapi.common.dto.TopicAuthorityDTO; + +public class AuthorityConverter { + public static AuthorityDO convert2AuthorityDO(TopicAuthorityDTO dto) { + AuthorityDO authorityDO = new AuthorityDO(); + authorityDO.setAppId(dto.getAppId()); + authorityDO.setClusterId(dto.getClusterId()); + authorityDO.setTopicName(dto.getTopicName()); + authorityDO.setAccess(dto.getAccess()); + return authorityDO; + } +} From 611f8b8865fabb9fcd785d1bac6fb3dff2bb4488 Mon Sep 17 00:00:00 2001 From: "tangcongfa_v@didichuxing.com" Date: Tue, 18 May 2021 18:25:03 +0800 Subject: [PATCH 31/64] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=85=8D=E9=A2=9D?= =?UTF-8?q?=E4=B8=8E=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../entity/dto/gateway/TopicQuotaDTO.java | 10 ++--- .../service/service/TopicManagerService.java | 8 ---- .../service/service/gateway/QuotaService.java | 8 ++++ .../gateway/impl/QuotaServiceImpl.java | 41 +++++++++++++++++++ .../service/impl/TopicManagerServiceImpl.java | 35 ---------------- .../openapi/common/dto/TopicAuthorityDTO.java | 8 ++-- .../thirdpart/ThirdPartTopicController.java | 10 +++-- 7 files changed, 65 insertions(+), 55 deletions(-) diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java index dd95a3ba..bff415a5 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java @@ -11,10 +11,10 @@ public class TopicQuotaDTO extends ClusterTopicDTO { @ApiModelProperty(value = "appId") private String appId; - @ApiModelProperty(value = "发送数据速率") + @ApiModelProperty(value = "发送数据速率B/s") private Long produceQuota; - @ApiModelProperty(value = "消费数据速率") + @ApiModelProperty(value = "消费数据速率B/s") private Long consumeQuota; public String getAppId() { @@ -42,9 +42,9 @@ public class TopicQuotaDTO extends ClusterTopicDTO { } public boolean paramLegal() { - if (ValidateUtils.isNull(clusterId) - || ValidateUtils.isNull(topicName) - || ValidateUtils.isNull(appId)) { + if (ValidateUtils.isNullOrLessThanZero(clusterId) + || ValidateUtils.isBlank(topicName) + || ValidateUtils.isBlank(appId)) { return false; } return true; diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java index 5e7e7a75..79524204 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java @@ -3,7 +3,6 @@ package com.xiaojukeji.kafka.manager.service.service; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.RdTopicBasic; -import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicAppData; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicBusinessInfo; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicDTO; @@ -125,13 +124,6 @@ public interface TopicManagerService { TopicBusinessInfo getTopicBusinessInfo(Long physicalClusterId, String topicName); - /** - * topic配额调整 - * @param topicQuota topic配额 - * @return - */ - ResultStatus addTopicQuota(TopicQuota topicQuota); - /** * topic权限调整 * @param authorityDO topic权限 diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/QuotaService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/QuotaService.java index 6a78e4f4..225c67b0 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/QuotaService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/QuotaService.java @@ -1,5 +1,6 @@ package com.xiaojukeji.kafka.manager.service.service.gateway; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; /** @@ -34,4 +35,11 @@ public interface QuotaService { TopicQuota getQuotaFromZk(Long clusterId, String topicName, String appId); Boolean modifyProduceQuota(Long clusterId, String topicName, String appId, Long produceQuota); + + /** + * topic配额调整 + * @param topicQuota topic配额 + * @return + */ + ResultStatus addTopicQuotaByAuthority(TopicQuota topicQuota); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/QuotaServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/QuotaServiceImpl.java index 2ce5facf..8baf61cc 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/QuotaServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/QuotaServiceImpl.java @@ -1,11 +1,16 @@ package com.xiaojukeji.kafka.manager.service.service.gateway.impl; +import com.xiaojukeji.kafka.manager.common.bizenum.TopicAuthorityEnum; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.utils.NumberUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.config.TopicQuotaData; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; +import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; +import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.service.service.TopicManagerService; import com.xiaojukeji.kafka.manager.service.strategy.AbstractAllocateQuotaStrategy; @@ -28,6 +33,12 @@ public class QuotaServiceImpl implements QuotaService { @Autowired private AbstractAllocateQuotaStrategy allocateQuotaStrategy; + @Autowired + private LogicalClusterMetadataManager logicalClusterMetadataManager; + + @Autowired + private AuthorityService authorityService; + @Override public int addTopicQuota(TopicQuota topicQuotaDO) { return KafkaZookeeperUtils.setTopicQuota( @@ -78,4 +89,34 @@ public class QuotaServiceImpl implements QuotaService { } return Boolean.TRUE; } + + @Override + public ResultStatus addTopicQuotaByAuthority(TopicQuota topicQuota) { + // 获取物理集群id + Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(topicQuota.getClusterId()); + if (ValidateUtils.isNull(physicalClusterId)) { + return ResultStatus.CLUSTER_NOT_EXIST; + } + // 权限判断(access 0:无权限, 1:读, 2:写, 3:读写，4:可管理) + AuthorityDO authority = authorityService.getAuthority(physicalClusterId, + topicQuota.getTopicName(), topicQuota.getAppId()); + if (ValidateUtils.isNull(authority) || authority.getAccess() == TopicAuthorityEnum.DENY.getCode()) { + return ResultStatus.USER_WITHOUT_AUTHORITY; + } + if (authority.getAccess() == TopicAuthorityEnum.READ.getCode()) { + // 可以消费 + topicQuota.setProduceQuota(null); + } + if (authority.getAccess() == TopicAuthorityEnum.WRITE.getCode()) { + // 可以生产 + topicQuota.setConsumeQuota(null); + } + // 设置物理集群id + topicQuota.setClusterId(physicalClusterId); + // 添加配额 + if (addTopicQuota(topicQuota) > 0) { + return ResultStatus.SUCCESS; + } + return ResultStatus.ZOOKEEPER_WRITE_FAILED; + } } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java index a5d459b4..4a8f501f 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java @@ -10,7 +10,6 @@ import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.RdTopicBasic; -import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.MineTopicSummary; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicAppData; import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicBusinessInfo; @@ -36,7 +35,6 @@ import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; -import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,9 +87,6 @@ public class TopicManagerServiceImpl implements TopicManagerService { @Autowired private OperateRecordService operateRecordService; - @Autowired - private QuotaService quotaService; - @Override public List listAll() { try { @@ -624,36 +619,6 @@ public class TopicManagerServiceImpl implements TopicManagerService { return topicBusinessInfo; } - @Override - public ResultStatus addTopicQuota(TopicQuota topicQuota) { - // 获取物理集群id - Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(topicQuota.getClusterId()); - if (ValidateUtils.isNull(physicalClusterId)) { - return ResultStatus.CLUSTER_NOT_EXIST; - } - // 权限判断(access 0:无权限, 1:读, 2:写, 3:读写，4:可管理) - AuthorityDO authority = authorityService.getAuthority(physicalClusterId, - topicQuota.getTopicName(), topicQuota.getAppId()); - if (ValidateUtils.isNull(authority) || authority.getAccess() == TopicAuthorityEnum.DENY.getCode()) { - return ResultStatus.USER_WITHOUT_AUTHORITY; - } - if (authority.getAccess() == TopicAuthorityEnum.READ.getCode()) { - // 可以消费 - topicQuota.setProduceQuota(null); - } - if (authority.getAccess() == TopicAuthorityEnum.WRITE.getCode()) { - // 可以生产 - topicQuota.setConsumeQuota(null); - } - // 设置物理集群id - topicQuota.setClusterId(physicalClusterId); - // 添加配额 - if (quotaService.addTopicQuota(topicQuota) > 0) { - return ResultStatus.SUCCESS; - } - return ResultStatus.MYSQL_ERROR; - } - @Override public ResultStatus addAuthority(AuthorityDO authorityDO) { // 查询该用户拥有的应用 diff --git a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java index b5394f6b..bb596989 100644 --- a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java +++ b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java @@ -32,10 +32,10 @@ public class TopicAuthorityDTO extends ClusterTopicDTO { @Override public boolean paramLegal() { - if (ValidateUtils.isNull(clusterId) - || ValidateUtils.isNull(topicName) - || ValidateUtils.isNull(appId) - || ValidateUtils.isNull(access)) { + if (ValidateUtils.isNullOrLessThanZero(clusterId) + || ValidateUtils.isBlank(topicName) + || ValidateUtils.isBlank(appId) + || ValidateUtils.isNullOrLessThanZero(access)) { return false; } return true; diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index 93907371..0fa615b5 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -21,6 +21,7 @@ import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; +import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; import com.xiaojukeji.kafka.manager.web.converters.AuthorityConverter; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter; @@ -56,6 +57,9 @@ public class ThirdPartTopicController { @Autowired private TopicManagerService topicManagerService; + @Autowired + private QuotaService quotaService; + @ApiOperation(value = "Topic元信息", notes = "LogX调用") @RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET) @ResponseBody @@ -139,18 +143,18 @@ public class ThirdPartTopicController { } @ApiOperation(value = "配额调整",notes = "配额调整") - @RequestMapping(value = "{topics/quota/add}",method = RequestMethod.POST) + @RequestMapping(value = "{topics/quota}",method = RequestMethod.POST) @ResponseBody public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { // 非空校验 if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } - return Result.buildFrom(topicManagerService.addTopicQuota(TopicQuota.buildFrom(dto))); + return Result.buildFrom(quotaService.addTopicQuotaByAuthority(TopicQuota.buildFrom(dto))); } @ApiOperation(value = "权限调整",notes = "权限调整") - @RequestMapping(value = "{topics/authority/add}",method = RequestMethod.POST) + @RequestMapping(value = "{topics/authority}",method = RequestMethod.POST) @ResponseBody public Result addAuthority(@RequestBody TopicAuthorityDTO dto) { //非空校验 From 7719339f2385f59bf606f268d4d8ac5793be4129 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 18 May 2021 20:18:08 +0800 Subject: [PATCH 32/64] split op util controller to topic controller and leader controller, and add authority controller, quota controller --- .../entity/dto/gateway/TopicQuotaDTO.java | 35 ++-- .../openapi/common/dto/TopicAuthorityDTO.java | 56 +++-- .../versionone/op/OpAuthorityController.java | 35 ++++ .../OpLeaderController.java} | 29 ++- .../api/versionone/op/OpQuotaController.java | 37 ++++ ...Controller.java => OpTopicController.java} | 194 +++++++----------- .../thirdpart/ThirdPartTopicController.java | 30 --- 7 files changed, 209 insertions(+), 207 deletions(-) create mode 100644 kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpAuthorityController.java rename kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/{thirdpart/ThirdPartOpUtilController.java => op/OpLeaderController.java} (69%) create mode 100644 kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpQuotaController.java rename kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/{OpUtilsController.java => OpTopicController.java} (69%) diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java index bff415a5..5719cd28 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/gateway/TopicQuotaDTO.java @@ -7,46 +7,41 @@ import io.swagger.annotations.ApiModelProperty; @ApiModel(description = "配额调整") public class TopicQuotaDTO extends ClusterTopicDTO { + @ApiModelProperty(value = "appId") + private String appId; - @ApiModelProperty(value = "appId") - private String appId; + @ApiModelProperty(value = "发送数据速率B/s") + private Long produceQuota; - @ApiModelProperty(value = "发送数据速率B/s") - private Long produceQuota; + @ApiModelProperty(value = "消费数据速率B/s") + private Long consumeQuota; - @ApiModelProperty(value = "消费数据速率B/s") - private Long consumeQuota; - - public String getAppId() { + public String getAppId() { return appId; } - public void setAppId(String appId) { + public void setAppId(String appId) { this.appId = appId; } - public Long getProduceQuota() { + public Long getProduceQuota() { return produceQuota; } - public void setProduceQuota(Long produceQuota) { + public void setProduceQuota(Long produceQuota) { this.produceQuota = produceQuota; } - public Long getConsumeQuota() { + public Long getConsumeQuota() { return consumeQuota; } - public void setConsumeQuota(Long consumeQuota) { + public void setConsumeQuota(Long consumeQuota) { this.consumeQuota = consumeQuota; } - public boolean paramLegal() { - if (ValidateUtils.isNullOrLessThanZero(clusterId) - || ValidateUtils.isBlank(topicName) - || ValidateUtils.isBlank(appId)) { - return false; + @Override + public boolean paramLegal() { + return !ValidateUtils.isNullOrLessThanZero(clusterId) && !ValidateUtils.isBlank(topicName) && !ValidateUtils.isBlank(appId); } - return true; - } } diff --git a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java index bb596989..30ceecb2 100644 --- a/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java +++ b/kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/TopicAuthorityDTO.java @@ -7,37 +7,33 @@ import io.swagger.annotations.ApiModelProperty; @ApiModel(description = "权限调整") public class TopicAuthorityDTO extends ClusterTopicDTO { + @ApiModelProperty(value = "appId") + private String appId; - @ApiModelProperty(value = "appId") - private String appId; + @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写, 4:可管理") + private Integer access; - @ApiModelProperty(value = "0:无权限, 1:读, 2:写, 3:读写, 4:可管理") - private Integer access; - - public String getAppId() { - return appId; - } - - public void setAppId(String appId) { - this.appId = appId; - } - - public Integer getAccess() { - return access; - } - - public void setAccess(Integer access) { - this.access = access; - } - - @Override - public boolean paramLegal() { - if (ValidateUtils.isNullOrLessThanZero(clusterId) - || ValidateUtils.isBlank(topicName) - || ValidateUtils.isBlank(appId) - || ValidateUtils.isNullOrLessThanZero(access)) { - return false; + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public Integer getAccess() { + return access; + } + + public void setAccess(Integer access) { + this.access = access; + } + + @Override + public boolean paramLegal() { + return !ValidateUtils.isNullOrLessThanZero(clusterId) + && !ValidateUtils.isBlank(topicName) + && !ValidateUtils.isBlank(appId) + && !ValidateUtils.isNullOrLessThanZero(access); } - return true; - } } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpAuthorityController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpAuthorityController.java new file mode 100644 index 00000000..bd415527 --- /dev/null +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpAuthorityController.java @@ -0,0 +1,35 @@ +package com.xiaojukeji.kafka.manager.web.api.versionone.op; + +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.openapi.common.dto.TopicAuthorityDTO; +import com.xiaojukeji.kafka.manager.service.service.TopicManagerService; +import com.xiaojukeji.kafka.manager.web.converters.AuthorityConverter; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +/** + * Authority操作相关接口 + * @author zengqiao + * @date 21/5/18 + */ +@Api(tags = "OP-Authority操作相关接口(REST)") +@RestController +public class OpAuthorityController { + @Autowired + private TopicManagerService topicManagerService; + + @ApiOperation(value = "权限调整",notes = "权限调整") + @PostMapping(value = "topic-authorities") + @ResponseBody + public Result addAuthority(@RequestBody TopicAuthorityDTO dto) { + //非空校验 + if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return Result.buildFrom(topicManagerService.addAuthority(AuthorityConverter.convert2AuthorityDO(dto))); + } +} diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartOpUtilController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpLeaderController.java similarity index 69% rename from kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartOpUtilController.java rename to kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpLeaderController.java index b0d8ffd6..6b344831 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartOpUtilController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpLeaderController.java @@ -1,11 +1,13 @@ -package com.xiaojukeji.kafka.manager.web.api.versionone.thirdpart; +package com.xiaojukeji.kafka.manager.web.api.versionone.op; import com.xiaojukeji.kafka.manager.common.bizenum.RebalanceDimensionEnum; +import com.xiaojukeji.kafka.manager.common.bizenum.TaskStatusEnum; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.dto.op.RebalanceDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; +import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.service.service.AdminService; @@ -16,22 +18,35 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; /** + * Leader操作[选举|切换]相关接口 * @author zengqiao - * @date 20/9/23 + * @date 21/5/18 */ -@Api(tags = "开放接口-OP相关接口(REST)") +@Api(tags = "OP-Leader操作相关接口(REST)") @RestController -@RequestMapping(ApiPrefix.API_V1_THIRD_PART_OP_PREFIX) -public class ThirdPartOpUtilController { - +@RequestMapping(ApiPrefix.API_V1_OP_PREFIX) +public class OpLeaderController { @Autowired private AdminService adminService; @Autowired private ClusterService clusterService; + @ApiOperation(value = "优先副本选举状态") + @RequestMapping(value = {"leaders/preferred-replica-election-status", "utils/rebalance-status"}, method = RequestMethod.GET) + @ResponseBody + public Result preferredReplicaElectStatus(@RequestParam("clusterId") Long clusterId) { + ClusterDO clusterDO = clusterService.getById(clusterId); + if (ValidateUtils.isNull(clusterDO)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + TaskStatusEnum statusEnum = adminService.preferredReplicaElectionStatus(clusterDO); + return new Result<>(JsonUtils.toJson(statusEnum)); + } + @ApiOperation(value = "优先副本选举") - @RequestMapping(value = "op/rebalance", method = RequestMethod.POST) + @RequestMapping(value = {"leaders/preferred-replica-election", "utils/rebalance"}, method = RequestMethod.POST) @ResponseBody public Result preferredReplicaElect(@RequestBody RebalanceDTO reqObj) { if (!reqObj.paramLegal()) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpQuotaController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpQuotaController.java new file mode 100644 index 00000000..7d9c70d7 --- /dev/null +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpQuotaController.java @@ -0,0 +1,37 @@ +package com.xiaojukeji.kafka.manager.web.api.versionone.op; + +import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; +import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.TopicQuotaDTO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +/** + * Quota操作相关接口 + * @author zengqiao + * @date 21/5/18 + */ +@Api(tags = "OP-Quota操作相关接口(REST)") +@RestController +@RequestMapping(ApiPrefix.API_V1_OP_PREFIX) +public class OpQuotaController { + @Autowired + private QuotaService quotaService; + + @ApiOperation(value = "配额调整",notes = "配额调整") + @RequestMapping(value = "topic-quotas",method = RequestMethod.POST) + @ResponseBody + public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { + if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { + // 非空校验 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return Result.buildFrom(quotaService.addTopicQuotaByAuthority(TopicQuota.buildFrom(dto))); + } +} diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpTopicController.java similarity index 69% rename from kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java rename to kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpTopicController.java index 6d9e7a74..bf7a1340 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpTopicController.java @@ -1,24 +1,23 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.op; -import com.xiaojukeji.kafka.manager.common.bizenum.RebalanceDimensionEnum; -import com.xiaojukeji.kafka.manager.common.bizenum.TaskStatusEnum; +import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; import com.xiaojukeji.kafka.manager.common.constant.Constant; import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.TopicOperationResult; import com.xiaojukeji.kafka.manager.common.entity.dto.ClusterTopicDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.op.RebalanceDTO; -import com.xiaojukeji.kafka.manager.common.entity.dto.op.topic.*; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.topic.TopicCreationDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.topic.TopicDeletionDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.topic.TopicExpansionDTO; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.topic.TopicModificationDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; +import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.service.service.AdminService; import com.xiaojukeji.kafka.manager.service.service.ClusterService; -import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; -import com.xiaojukeji.kafka.manager.common.utils.SpringTool; import com.xiaojukeji.kafka.manager.service.service.TopicManagerService; -import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; -import com.xiaojukeji.kafka.manager.common.entity.TopicOperationResult; import com.xiaojukeji.kafka.manager.service.utils.TopicCommands; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -30,25 +29,25 @@ import java.util.List; import java.util.Properties; /** - * 运维工具类 + * Topic操作相关接口 * @author zengqiao - * @date 20/4/2 + * @date 21/5/18 */ -@Api(tags = "OP-Utils相关接口(REST)") +@Api(tags = "OP-Topic操作相关接口(REST)") @RestController @RequestMapping(ApiPrefix.API_V1_OP_PREFIX) -public class OpUtilsController { - @Autowired - private ClusterService clusterService; - +public class OpTopicController { @Autowired private AdminService adminService; + @Autowired + private ClusterService clusterService; + @Autowired private TopicManagerService topicManagerService; @ApiOperation(value = "创建Topic") - @RequestMapping(value = {"utils/topics"}, method = RequestMethod.POST) + @RequestMapping(value = {"topics", "utils/topics"}, method = RequestMethod.POST) @ResponseBody public Result createCommonTopic(@RequestBody TopicCreationDTO dto) { Result rc = checkParamAndGetClusterDO(dto); @@ -76,8 +75,66 @@ public class OpUtilsController { return Result.buildFrom(rs); } + @ApiOperation(value = "Topic删除", notes = "单次不允许超过10个Topic") + @RequestMapping(value = {"topics", "utils/topics"}, method = RequestMethod.DELETE) + @ResponseBody + public Result> deleteTopics(@RequestBody List dtoList) { + if (ValidateUtils.isNull(dtoList) || dtoList.size() > Constant.MAX_TOPIC_OPERATION_SIZE_PER_REQUEST) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + String operator = SpringTool.getUserName(); + + List resultList = new ArrayList<>(); + for (TopicDeletionDTO dto: dtoList) { + Result rc = checkParamAndGetClusterDO(dto); + if (rc.getCode() != ResultStatus.SUCCESS.getCode()) { + resultList.add(TopicOperationResult.buildFrom(dto.getClusterId(), dto.getTopicName(), rc)); + continue; + } + + // 参数检查合法, 开始删除Topic + ResultStatus statusEnum = adminService.deleteTopic(rc.getData(), dto.getTopicName(), operator); + resultList.add(TopicOperationResult.buildFrom(dto.getClusterId(), dto.getTopicName(), statusEnum)); + } + + for (TopicOperationResult operationResult: resultList) { + if (!Constant.SUCCESS.equals(operationResult.getCode())) { + return Result.buildFrom(ResultStatus.OPERATION_FAILED, resultList); + } + } + return new Result<>(resultList); + } + + @ApiOperation(value = "修改Topic", notes = "") + @RequestMapping(value = {"topics", "utils/topics"}, method = RequestMethod.PUT) + @ResponseBody + public Result modifyTopic(@RequestBody TopicModificationDTO dto) { + Result rc = checkParamAndGetClusterDO(dto); + if (rc.getCode() != ResultStatus.SUCCESS.getCode()) { + return rc; + } + + ClusterDO clusterDO = rc.getData(); + + // 获取属性 + Properties properties = dto.getProperties(); + if (ValidateUtils.isNull(properties)) { + properties = new Properties(); + } + properties.put(KafkaConstant.RETENTION_MS_KEY, String.valueOf(dto.getRetentionTime())); + + // 操作修改 + String operator = SpringTool.getUserName(); + ResultStatus rs = TopicCommands.modifyTopicConfig(clusterDO, dto.getTopicName(), properties); + if (!ResultStatus.SUCCESS.equals(rs)) { + return Result.buildFrom(rs); + } + topicManagerService.modifyTopicByOp(dto.getClusterId(), dto.getTopicName(), dto.getAppId(), dto.getDescription(), operator); + return new Result(); + } + @ApiOperation(value = "Topic扩分区", notes = "") - @RequestMapping(value = {"utils/expand-partitions"}, method = RequestMethod.PUT) + @RequestMapping(value = {"topics/expand-partitions", "utils/expand-partitions"}, method = RequestMethod.PUT) @ResponseBody public Result> expandTopics(@RequestBody List dtoList) { if (ValidateUtils.isNull(dtoList) || dtoList.size() > Constant.MAX_TOPIC_OPERATION_SIZE_PER_REQUEST) { @@ -112,108 +169,6 @@ public class OpUtilsController { return new Result<>(resultList); } - @ApiOperation(value = "Topic删除", notes = "单次不允许超过10个Topic") - @RequestMapping(value = {"utils/topics"}, method = RequestMethod.DELETE) - @ResponseBody - public Result> deleteTopics(@RequestBody List dtoList) { - if (ValidateUtils.isNull(dtoList) || dtoList.size() > Constant.MAX_TOPIC_OPERATION_SIZE_PER_REQUEST) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - String operator = SpringTool.getUserName(); - - List resultList = new ArrayList<>(); - for (TopicDeletionDTO dto: dtoList) { - Result rc = checkParamAndGetClusterDO(dto); - if (rc.getCode() != ResultStatus.SUCCESS.getCode()) { - resultList.add(TopicOperationResult.buildFrom(dto.getClusterId(), dto.getTopicName(), rc)); - continue; - } - - // 参数检查合法, 开始删除Topic - ResultStatus statusEnum = adminService.deleteTopic(rc.getData(), dto.getTopicName(), operator); - resultList.add(TopicOperationResult.buildFrom(dto.getClusterId(), dto.getTopicName(), statusEnum)); - } - - for (TopicOperationResult operationResult: resultList) { - if (!Constant.SUCCESS.equals(operationResult.getCode())) { - return Result.buildFrom(ResultStatus.OPERATION_FAILED, resultList); - } - } - return new Result<>(resultList); - } - - @ApiOperation(value = "修改Topic", notes = "") - @RequestMapping(value = {"utils/topics"}, method = RequestMethod.PUT) - @ResponseBody - public Result modifyTopic(@RequestBody TopicModificationDTO dto) { - Result rc = checkParamAndGetClusterDO(dto); - if (rc.getCode() != ResultStatus.SUCCESS.getCode()) { - return rc; - } - - ClusterDO clusterDO = rc.getData(); - - // 获取属性 - Properties properties = dto.getProperties(); - if (ValidateUtils.isNull(properties)) { - properties = new Properties(); - } - properties.put(KafkaConstant.RETENTION_MS_KEY, String.valueOf(dto.getRetentionTime())); - - // 操作修改 - String operator = SpringTool.getUserName(); - ResultStatus rs = TopicCommands.modifyTopicConfig(clusterDO, dto.getTopicName(), properties); - if (!ResultStatus.SUCCESS.equals(rs)) { - return Result.buildFrom(rs); - } - topicManagerService.modifyTopicByOp(dto.getClusterId(), dto.getTopicName(), dto.getAppId(), dto.getDescription(), operator); - return new Result(); - } - - @ApiOperation(value = "优先副本选举状态") - @RequestMapping(value = "utils/rebalance-status", method = RequestMethod.GET) - @ResponseBody - public Result preferredReplicaElectStatus(@RequestParam("clusterId") Long clusterId) { - ClusterDO clusterDO = clusterService.getById(clusterId); - if (ValidateUtils.isNull(clusterDO)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - - TaskStatusEnum statusEnum = adminService.preferredReplicaElectionStatus(clusterDO); - return new Result<>(JsonUtils.toJson(statusEnum)); - } - - @ApiOperation(value = "优先副本选举") - @RequestMapping(value = "utils/rebalance", method = RequestMethod.POST) - @ResponseBody - public Result preferredReplicaElect(@RequestBody RebalanceDTO reqObj) { - if (!reqObj.paramLegal()) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - ClusterDO clusterDO = clusterService.getById(reqObj.getClusterId()); - if (ValidateUtils.isNull(clusterDO)) { - return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); - } - - ResultStatus rs = null; - if (RebalanceDimensionEnum.CLUSTER.getCode().equals(reqObj.getDimension())) { - // 按照Cluster纬度均衡 - rs = adminService.preferredReplicaElection(clusterDO, SpringTool.getUserName()); - } else if (RebalanceDimensionEnum.BROKER.getCode().equals(reqObj.getDimension())) { - // 按照Broker纬度均衡 - rs = adminService.preferredReplicaElection(clusterDO, reqObj.getBrokerId(), SpringTool.getUserName()); - } else if (RebalanceDimensionEnum.TOPIC.getCode().equals(reqObj.getDimension())) { - // 按照Topic纬度均衡 - rs = adminService.preferredReplicaElection(clusterDO, reqObj.getTopicName(), SpringTool.getUserName()); - } else if (RebalanceDimensionEnum.PARTITION.getCode().equals(reqObj.getDimension())) { - // 按照Partition纬度均衡 - rs = adminService.preferredReplicaElection(clusterDO, reqObj.getTopicName(), reqObj.getPartitionId(), SpringTool.getUserName()); - } else { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return Result.buildFrom(rs); - } - private Result checkParamAndGetClusterDO(ClusterTopicDTO dto) { if (!dto.paramLegal()) { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); @@ -226,4 +181,3 @@ public class OpUtilsController { return new Result<>(clusterDO); } } - diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java index 0fa615b5..a84c01af 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartTopicController.java @@ -5,8 +5,6 @@ import com.xiaojukeji.kafka.manager.common.constant.Constant; import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; -import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.TopicQuota; -import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.TopicQuotaDTO; import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics; import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadataVO; @@ -14,15 +12,12 @@ import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGro import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicAuthorizedAppVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicRequestTimeDetailVO; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; -import com.xiaojukeji.kafka.manager.openapi.common.dto.TopicAuthorityDTO; import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; -import com.xiaojukeji.kafka.manager.service.service.gateway.QuotaService; -import com.xiaojukeji.kafka.manager.web.converters.AuthorityConverter; import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter; import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter; import com.xiaojukeji.kafka.manager.web.converters.TopicModelConverter; @@ -57,9 +52,6 @@ public class ThirdPartTopicController { @Autowired private TopicManagerService topicManagerService; - @Autowired - private QuotaService quotaService; - @ApiOperation(value = "Topic元信息", notes = "LogX调用") @RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET) @ResponseBody @@ -141,26 +133,4 @@ public class ThirdPartTopicController { topicManagerService.getTopicAuthorizedApps(physicalClusterId, topicName)) ); } - - @ApiOperation(value = "配额调整",notes = "配额调整") - @RequestMapping(value = "{topics/quota}",method = RequestMethod.POST) - @ResponseBody - public Result addTopicQuota(@RequestBody TopicQuotaDTO dto) { - // 非空校验 - if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return Result.buildFrom(quotaService.addTopicQuotaByAuthority(TopicQuota.buildFrom(dto))); - } - - @ApiOperation(value = "权限调整",notes = "权限调整") - @RequestMapping(value = "{topics/authority}",method = RequestMethod.POST) - @ResponseBody - public Result addAuthority(@RequestBody TopicAuthorityDTO dto) { - //非空校验 - if (ValidateUtils.isNull(dto) || !dto.paramLegal()) { - return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); - } - return Result.buildFrom(topicManagerService.addAuthority(AuthorityConverter.convert2AuthorityDO(dto))); - } } From b8c07a966f39875aadd1ceca304efc3e66534fcb Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 18 May 2021 21:00:20 +0800 Subject: [PATCH 33/64] bump version to v2.4.1 --- build.sh | 2 +- kafka-manager-console/package.json | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 9c107907..ba2c3c44 100644 --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ cd $workspace ## constant OUTPUT_DIR=./output -KM_VERSION=2.4.0 +KM_VERSION=2.4.1 APP_NAME=kafka-manager APP_DIR=${APP_NAME}-${KM_VERSION} diff --git a/kafka-manager-console/package.json b/kafka-manager-console/package.json index 920fa613..c0fa003e 100644 --- a/kafka-manager-console/package.json +++ b/kafka-manager-console/package.json @@ -1,6 +1,6 @@ { "name": "logi-kafka", - "version": "2.3.1", + "version": "2.4.1", "description": "", "scripts": { "start": "webpack-dev-server", diff --git a/pom.xml b/pom.xml index 9d4a5d30..bbbbd054 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ - 2.4.0-SNAPSHOT + 2.4.1-SNAPSHOT 2.7.0 1.5.13 From 49368aaf76aa259c7465b24d16b8aaf15ec66c97 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 20 May 2021 18:17:30 +0800 Subject: [PATCH 34/64] bump tomcat version to 8.5.56 --- kafka-manager-web/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-manager-web/pom.xml b/kafka-manager-web/pom.xml index 849eb304..f6d315fb 100644 --- a/kafka-manager-web/pom.xml +++ b/kafka-manager-web/pom.xml @@ -19,7 +19,7 @@ 2.1.1.RELEASE 5.1.3.RELEASE false - 8.5.37 + 8.5.56 diff --git a/pom.xml b/pom.xml index 9d4a5d30..2da00d65 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 1.8 UTF-8 UTF-8 - 8.5.37 + 8.5.56 From 72e743dfd103d4d1a88495a963035e60d77f3f99 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 20 May 2021 20:35:23 +0800 Subject: [PATCH 35/64] fix title version --- kafka-manager-console/src/container/header/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-manager-console/src/container/header/index.tsx b/kafka-manager-console/src/container/header/index.tsx index ebda9760..3f26949b 100644 --- a/kafka-manager-console/src/container/header/index.tsx +++ b/kafka-manager-console/src/container/header/index.tsx @@ -145,7 +145,7 @@ export const Header = observer((props: IHeader) => {
Kafka Manager - v2.4.0 + v2.4.1 {/* 添加版本超链接 */}
From da7ad8b44a5a605fe37ddddb1c0b8c91db4ba0b9 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Fri, 21 May 2021 10:20:09 +0800 Subject: [PATCH 36/64] bump tomcat version to 8.5.66 --- kafka-manager-web/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-manager-web/pom.xml b/kafka-manager-web/pom.xml index f6d315fb..17504ca7 100644 --- a/kafka-manager-web/pom.xml +++ b/kafka-manager-web/pom.xml @@ -19,7 +19,7 @@ 2.1.1.RELEASE 5.1.3.RELEASE false - 8.5.56 + 8.5.66 diff --git a/pom.xml b/pom.xml index 2da00d65..4a7787de 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 1.8 UTF-8 UTF-8 - 8.5.56 + 8.5.66 From 84df377516ae810d66afe502ed11745a12e9ecd8 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Fri, 21 May 2021 10:45:10 +0800 Subject: [PATCH 37/64] bump version to v2.4.2 and add release notes --- Releases_Notes.md | 19 ++++++++++++++++++- build.sh | 2 +- kafka-manager-console/package.json | 2 +- .../src/container/header/index.tsx | 2 +- pom.xml | 2 +- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Releases_Notes.md b/Releases_Notes.md index 471ce227..50a31c3b 100644 --- a/Releases_Notes.md +++ b/Releases_Notes.md @@ -7,9 +7,26 @@ --- +## v2.4.1+ + +版本上线时间：2021-05-21 + +### 能力提升 +- 增加直接增加权限和配额的接口(v2.4.1) +- 增加接口调用可绕过登录的功能(v2.4.1) + +### 体验优化 +- tomcat 版本提升至8.5.66(v2.4.2) +- op接口优化，拆分util接口为topic、leader两类接口(v2.4.1) +- 简化Gateway配置的Key长度(v2.4.1) + +### bug修复 +- 修复页面展示版本错误问题(v2.4.2) + + ## v2.4.0 -版本上线时间：2021-04-26 +版本上线时间：2021-05-18 ### 能力提升 diff --git a/build.sh b/build.sh index ba2c3c44..b07c6623 100644 --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ cd $workspace ## constant OUTPUT_DIR=./output -KM_VERSION=2.4.1 +KM_VERSION=2.4.2 APP_NAME=kafka-manager APP_DIR=${APP_NAME}-${KM_VERSION} diff --git a/kafka-manager-console/package.json b/kafka-manager-console/package.json index c0fa003e..53004b5b 100644 --- a/kafka-manager-console/package.json +++ b/kafka-manager-console/package.json @@ -1,6 +1,6 @@ { "name": "logi-kafka", - "version": "2.4.1", + "version": "v2.4.2", "description": "", "scripts": { "start": "webpack-dev-server", diff --git a/kafka-manager-console/src/container/header/index.tsx b/kafka-manager-console/src/container/header/index.tsx index 3f26949b..e12e397c 100644 --- a/kafka-manager-console/src/container/header/index.tsx +++ b/kafka-manager-console/src/container/header/index.tsx @@ -145,7 +145,7 @@ export const Header = observer((props: IHeader) => {
Kafka Manager - v2.4.1 + v2.4.2 {/* 添加版本超链接 */}
diff --git a/pom.xml b/pom.xml index 68a9399b..51fa20e0 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ - 2.4.1-SNAPSHOT + 2.4.2-SNAPSHOT 2.7.0 1.5.13 From c977ce5690959501d9c031f11670574d4af6b492 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 25 May 2021 10:27:27 +0800 Subject: [PATCH 38/64] =?UTF-8?q?heartbeat=E8=A1=A8=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=9B=B4=E6=96=B0=E6=97=B6=E9=97=B4=E4=BB=8EMySQL?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E7=94=9F=E6=88=90=E8=B0=83=E6=95=B4=E4=B8=BA?= =?UTF-8?q?Logi-KM=E7=9A=84=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kafka-manager-dao/src/main/resources/mapper/HeartbeatDao.xml | 2 +- .../com/xiaojukeji/kafka/manager/task/component/Heartbeat.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kafka-manager-dao/src/main/resources/mapper/HeartbeatDao.xml b/kafka-manager-dao/src/main/resources/mapper/HeartbeatDao.xml index 900e13a9..bea686ff 100644 --- a/kafka-manager-dao/src/main/resources/mapper/HeartbeatDao.xml +++ b/kafka-manager-dao/src/main/resources/mapper/HeartbeatDao.xml @@ -11,7 +11,7 @@ - REPLACE heartbeat (ip, hostname) VALUES (#{ip}, #{hostname}) + REPLACE heartbeat (ip, hostname, modify_time) VALUES (#{ip}, #{hostname}, #{modifyTime})