开放接口&近期BUG修复

This commit is contained in:
zengqiao
2020-10-26 11:17:45 +08:00
parent 8b153113ff
commit a77242e66c
62 changed files with 12138 additions and 423 deletions

View File

@@ -1,5 +1,7 @@
package com.xiaojukeji.kafka.manager.web;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -18,10 +20,13 @@ import org.springframework.scheduling.annotation.EnableScheduling;
@EnableAutoConfiguration
@SpringBootApplication(scanBasePackages = {"com.xiaojukeji.kafka.manager"})
public class MainApplication {
private static final Logger LOGGER = LoggerFactory.getLogger(MainApplication.class);
public static void main(String[] args) {
try {
SpringApplication sa = new SpringApplication(MainApplication.class);
sa.run(args);
LOGGER.info("MainApplication started");
} catch (Exception e) {
e.printStackTrace();
}

View File

@@ -3,7 +3,7 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.gateway;
import com.alibaba.fastjson.JSON;
import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
import com.xiaojukeji.kafka.manager.common.entity.DeprecatedResponseResult;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.TopicConnectionDTO;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.service.service.gateway.TopicConnectionService;
@@ -34,19 +34,19 @@ public class GatewayHeartbeatController {
@ApiOperation(value = "连接信息上报入口", notes = "Broker主动上报信息")
@RequestMapping(value = "heartbeat/survive-user", method = RequestMethod.POST)
@ResponseBody
public DeprecatedResponseResult receiveTopicConnections(@RequestParam("clusterId") String clusterId,
@RequestParam("brokerId") String brokerId,
@RequestBody List<TopicConnectionDTO> dtoList) {
public Result receiveTopicConnections(@RequestParam("clusterId") String clusterId,
@RequestParam("brokerId") String brokerId,
@RequestBody List<TopicConnectionDTO> dtoList) {
try {
if (ValidateUtils.isEmptyList(dtoList)) {
return DeprecatedResponseResult.success("success");
return Result.buildSuc();
}
topicConnectionService.batchAdd(dtoList);
return DeprecatedResponseResult.success("success");
return Result.buildSuc();
} catch (Exception e) {
LOGGER.error("receive topic connections failed, clusterId:{} brokerId:{} req:{}",
clusterId, brokerId, JSON.toJSONString(dtoList), e);
}
return DeprecatedResponseResult.failure("fail");
return Result.buildFailure("fail");
}
}

View File

@@ -2,7 +2,7 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.gateway;
import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
import com.xiaojukeji.kafka.manager.common.entity.DeprecatedResponseResult;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.utils.ListUtils;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.TopicReportDO;
@@ -35,15 +35,15 @@ public class GatewayReportController {
@ApiOperation(value = "查询开启JMX采集的Topic", notes = "")
@RequestMapping(value = "report/jmx/topics", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult getJmxReportTopics(@RequestParam("clusterId") Long clusterId) {
public Result getJmxReportTopics(@RequestParam("clusterId") Long clusterId) {
List<TopicReportDO> doList = topicReportService.getNeedReportTopic(clusterId);
if (ValidateUtils.isEmptyList(doList)) {
return DeprecatedResponseResult.success();
if (ValidateUtils.isNull(doList)) {
doList = new ArrayList<>();
}
List<String> topicNameList = new ArrayList<>();
for (TopicReportDO elem: doList) {
topicNameList.add(elem.getTopicName());
}
return DeprecatedResponseResult.success(ListUtils.strList2String(topicNameList));
return Result.buildSuc(ListUtils.strList2String(topicNameList));
}
}

View File

@@ -3,7 +3,8 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.gateway;
import com.alibaba.fastjson.JSON;
import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
import com.xiaojukeji.kafka.manager.common.entity.DeprecatedResponseResult;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.KafkaAclSearchDTO;
import com.xiaojukeji.kafka.manager.common.entity.dto.gateway.KafkaUserSearchDTO;
import com.xiaojukeji.kafka.manager.common.entity.vo.gateway.KafkaSecurityVO;
@@ -40,9 +41,9 @@ public class GatewaySecurityController {
@ApiOperation(value = "Kafka用户查询", notes = "")
@RequestMapping(value = "security/users", method = RequestMethod.POST)
@ResponseBody
public DeprecatedResponseResult<String> getKafkaUsers(@RequestBody KafkaUserSearchDTO dto) {
public Result<String> getKafkaUsers(@RequestBody KafkaUserSearchDTO dto) {
if (ValidateUtils.isNull(dto) || !dto.paramLegal()) {
return DeprecatedResponseResult.failure("invalid request");
return Result.buildFrom(ResultStatus.GATEWAY_INVALID_REQUEST);
}
try {
@@ -50,16 +51,16 @@ public class GatewaySecurityController {
dto.getStart(),
dto.getEnd().equals(0L)? System.currentTimeMillis(): dto.getEnd()
);
if (ValidateUtils.isEmptyList(doList)) {
return DeprecatedResponseResult.success();
if (ValidateUtils.isNull(doList)) {
doList = new ArrayList<>();
}
KafkaSecurityVO vo = new KafkaSecurityVO();
vo.setRows(new ArrayList<>(GatewayModelConverter.convert2KafkaUserVOList(doList)));
return DeprecatedResponseResult.success(JSON.toJSONString(vo));
return Result.buildSuc(JSON.toJSONString(vo));
} catch (Exception e) {
LOGGER.error("get kafka users failed, req:{}.", dto, e);
return DeprecatedResponseResult.failure("get kafka users exception");
return Result.buildFrom(ResultStatus.MYSQL_ERROR);
}
}
@@ -67,9 +68,9 @@ public class GatewaySecurityController {
@ApiOperation(value = "Kafka用户权限查询", notes = "")
@RequestMapping(value = "security/acls", method = RequestMethod.POST)
@ResponseBody
public DeprecatedResponseResult<String> getKafkaAcls(@RequestBody KafkaAclSearchDTO dto) {
public Result<String> getKafkaAcls(@RequestBody KafkaAclSearchDTO dto) {
if (ValidateUtils.isNull(dto) || !dto.paramLegal()) {
return DeprecatedResponseResult.failure("invalid request");
return Result.buildFrom(ResultStatus.GATEWAY_INVALID_REQUEST);
}
try {
@@ -78,16 +79,16 @@ public class GatewaySecurityController {
dto.getStart(),
dto.getEnd().equals(0L)? System.currentTimeMillis(): dto.getEnd()
);
if (ValidateUtils.isEmptyList(doList)) {
return DeprecatedResponseResult.success();
if (ValidateUtils.isNull(doList)) {
doList = new ArrayList<>();
}
KafkaSecurityVO vo = new KafkaSecurityVO();
vo.setRows(new ArrayList<>(GatewayModelConverter.convert2KafkaAclVOList(doList)));
return DeprecatedResponseResult.success(JSON.toJSONString(vo));
return Result.buildSuc(JSON.toJSONString(vo));
} catch (Exception e) {
LOGGER.error("get kafka acls failed, req:{}.", dto, e);
return DeprecatedResponseResult.failure("get kafka acls exception");
return Result.buildFrom(ResultStatus.MYSQL_ERROR);
}
}
}

View File

@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSON;
import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
import com.xiaojukeji.kafka.manager.common.bizenum.gateway.GatewayConfigKeyEnum;
import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
import com.xiaojukeji.kafka.manager.common.entity.DeprecatedResponseResult;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.*;
import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO;
import com.xiaojukeji.kafka.manager.common.entity.vo.gateway.GatewayConfigVO;
@@ -54,33 +54,27 @@ public class GatewayServiceDiscoveryController {
@ApiOperation(value = "获取集群服务地址", notes = "")
@RequestMapping(value = "discovery/init", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult<String> getAllKafkaBootstrapServers() {
public Result<String> getAllKafkaBootstrapServers() {
KafkaBootstrapServerConfig config =
gatewayConfigService.getKafkaBootstrapServersConfig(Long.MIN_VALUE);
if (ValidateUtils.isNull(config) || ValidateUtils.isNull(config.getClusterIdBootstrapServersMap())) {
return DeprecatedResponseResult.failure("call init kafka bootstrap servers failed");
return Result.buildFailure("call init kafka bootstrap servers failed");
}
if (config.getClusterIdBootstrapServersMap().isEmpty()) {
return DeprecatedResponseResult.success();
}
return DeprecatedResponseResult.success(JSON.toJSONString(config.getClusterIdBootstrapServersMap()));
return Result.buildSuc(JSON.toJSONString(config.getClusterIdBootstrapServersMap()));
}
@ApiLevel(level = ApiLevelContent.LEVEL_IMPORTANT_2)
@ApiOperation(value = "获取集群服务地址", notes = "")
@RequestMapping(value = "discovery/update", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult getBootstrapServersIfNeeded(@RequestParam("versionNumber") long versionNumber) {
public Result<String> getBootstrapServersIfNeeded(@RequestParam("versionNumber") long versionNumber) {
KafkaBootstrapServerConfig config =
gatewayConfigService.getKafkaBootstrapServersConfig(versionNumber);
if (ValidateUtils.isNull(config) || ValidateUtils.isNull(config.getClusterIdBootstrapServersMap())) {
return DeprecatedResponseResult.failure("call update kafka bootstrap servers failed");
return Result.buildFailure("call update kafka bootstrap servers failed");
}
if (config.getClusterIdBootstrapServersMap().isEmpty()) {
return DeprecatedResponseResult.success();
}
return DeprecatedResponseResult.success(JSON.toJSONString(new GatewayConfigVO(
return Result.buildSuc(JSON.toJSONString(new GatewayConfigVO(
String.valueOf(config.getVersion()),
JSON.toJSONString(config.getClusterIdBootstrapServersMap())
)));
@@ -90,15 +84,13 @@ public class GatewayServiceDiscoveryController {
@ApiOperation(value = "最大并发请求数", notes = "")
@RequestMapping(value = "discovery/max-request-num", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult getMaxRequestNum(@RequestParam("versionNumber") long versionNumber) {
public Result<String> getMaxRequestNum(@RequestParam("versionNumber") long versionNumber) {
RequestQueueConfig config = gatewayConfigService.getRequestQueueConfig(versionNumber);
if (ValidateUtils.isNull(config)) {
return DeprecatedResponseResult.failure("call get request queue size config failed");
return Result.buildFailure("call get request queue size config failed");
}
if (ValidateUtils.isNull(config.getMaxRequestQueueSize())) {
return DeprecatedResponseResult.success();
}
return DeprecatedResponseResult.success(JSON.toJSONString(
return Result.buildSuc(JSON.toJSONString(
new GatewayConfigVO(
String.valueOf(config.getVersion()),
String.valueOf(config.getMaxRequestQueueSize())
@@ -110,15 +102,13 @@ public class GatewayServiceDiscoveryController {
@ApiOperation(value = "最大APP请求速率", notes = "")
@RequestMapping(value = "discovery/appId-rate", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult getAppIdRate(@RequestParam("versionNumber") long versionNumber) {
public Result<String> getAppIdRate(@RequestParam("versionNumber") long versionNumber) {
AppRateConfig config = gatewayConfigService.getAppRateConfig(versionNumber);
if (ValidateUtils.isNull(config)) {
return DeprecatedResponseResult.failure("call get app rate config failed");
return Result.buildFailure("call get app rate config failed");
}
if (ValidateUtils.isNull(config.getAppRateLimit())) {
return DeprecatedResponseResult.success();
}
return DeprecatedResponseResult.success(JSON.toJSONString(
return Result.buildSuc(JSON.toJSONString(
new GatewayConfigVO(
String.valueOf(config.getVersion()),
String.valueOf(config.getAppRateLimit())
@@ -130,15 +120,12 @@ public class GatewayServiceDiscoveryController {
@ApiOperation(value = "最大IP请求速率", notes = "")
@RequestMapping(value = "discovery/ip-rate", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult getIpRate(@RequestParam("versionNumber") long versionNumber) {
public Result getIpRate(@RequestParam("versionNumber") long versionNumber) {
IpRateConfig config = gatewayConfigService.getIpRateConfig(versionNumber);
if (ValidateUtils.isNull(config)) {
return DeprecatedResponseResult.failure("call get ip rate config failed");
return Result.buildFailure("call get ip rate config failed");
}
if (ValidateUtils.isNull(config.getIpRateLimit())) {
return DeprecatedResponseResult.success();
}
return DeprecatedResponseResult.success(JSON.toJSONString(
return Result.buildSuc(JSON.toJSONString(
new GatewayConfigVO(
String.valueOf(config.getVersion()),
String.valueOf(config.getIpRateLimit())
@@ -150,15 +137,11 @@ public class GatewayServiceDiscoveryController {
@ApiOperation(value = "最大SP请求速率", notes = "")
@RequestMapping(value = "discovery/sp-limit", method = RequestMethod.GET)
@ResponseBody
public DeprecatedResponseResult getSpLimit(@RequestParam("versionNumber") long versionNumber) {
public Result<String> getSpLimit(@RequestParam("versionNumber") long versionNumber) {
SpRateConfig config =
gatewayConfigService.getSpRateConfig(versionNumber);
if (ValidateUtils.isNull(config) || ValidateUtils.isNull(config.getSpRateMap())) {
return DeprecatedResponseResult.failure("call update kafka bootstrap servers failed");
}
if (config.getSpRateMap().isEmpty()) {
return DeprecatedResponseResult.success();
return Result.buildFailure("call update kafka bootstrap servers failed");
}
List<String> strList = new ArrayList<>();
@@ -166,7 +149,7 @@ public class GatewayServiceDiscoveryController {
strList.add(entry.getKey() + "#" + String.valueOf(entry.getValue()));
}
return DeprecatedResponseResult.success(JSON.toJSONString(new GatewayConfigVO(
return Result.buildSuc(JSON.toJSONString(new GatewayConfigVO(
String.valueOf(config.getVersion()),
ListUtils.strList2String(strList)
)));

View File

@@ -0,0 +1,49 @@
package com.xiaojukeji.kafka.manager.web.api.versionone.thirdpart;
import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
import com.xiaojukeji.kafka.manager.common.constant.SystemCodeConstant;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.vo.normal.app.AppVO;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.service.service.gateway.AppService;
import com.xiaojukeji.kafka.manager.web.converters.AppConverter;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* @author zengqiao
* @date 20/9/23
*/
@Api(tags = "开放接口-App相关接口(REST)")
@RestController
@RequestMapping(ApiPrefix.API_V1_THIRD_PART_PREFIX)
public class ThirdPartAppController {
private final static Logger LOGGER = LoggerFactory.getLogger(ThirdPartAppController.class);
@Autowired
private AppService appService;
@ApiOperation(value = "查询负责的应用", notes = "")
@RequestMapping(value = "principal-apps/{principal}/basic-info", method = RequestMethod.GET)
@ResponseBody
public Result<List<AppVO>> searchPrincipalApps(@PathVariable("principal") String principal,
@RequestParam("system-code") String systemCode) {
LOGGER.info("search principal-apps, principal:{} systemCode:{}.", principal, systemCode);
if (ValidateUtils.isBlank(principal) || ValidateUtils.isBlank(systemCode)) {
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
}
if (!SystemCodeConstant.KAFKA_MANAGER.equals(systemCode)) {
return Result.buildFrom(ResultStatus.OPERATION_FORBIDDEN);
}
return new Result<>(AppConverter.convert2AppVOList(
appService.getByPrincipal(principal)
));
}
}

View File

@@ -0,0 +1,109 @@
package com.xiaojukeji.kafka.manager.web.api.versionone.thirdpart;
import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.metrics.BrokerMetrics;
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
import com.xiaojukeji.kafka.manager.common.entity.pojo.RegionDO;
import com.xiaojukeji.kafka.manager.openapi.common.vo.ThirdPartBrokerOverviewVO;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata;
import com.xiaojukeji.kafka.manager.openapi.common.vo.BrokerRegionVO;
import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
import com.xiaojukeji.kafka.manager.service.service.BrokerService;
import com.xiaojukeji.kafka.manager.service.service.ClusterService;
import com.xiaojukeji.kafka.manager.service.service.RegionService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @author zengqiao
* @date 20/9/9
*/
@Api(tags = "开放接口-Broker相关接口(REST)")
@RestController
@RequestMapping(ApiPrefix.API_V1_THIRD_PART_PREFIX)
public class ThirdPartBrokerController {
@Autowired
private BrokerService brokerService;
@Autowired
private RegionService regionService;
@Autowired
private ClusterService clusterService;
@ApiOperation(value = "Broker信息概览", notes = "")
@RequestMapping(value = "{clusterId}/brokers/{brokerId}/overview", method = RequestMethod.GET)
@ResponseBody
public Result<ThirdPartBrokerOverviewVO> getBrokerOverview(@PathVariable Long clusterId,
@PathVariable Integer brokerId) {
BrokerMetadata brokerMetadata = PhysicalClusterMetadataManager.getBrokerMetadata(clusterId, brokerId);
if (ValidateUtils.isNull(brokerMetadata)) {
return Result.buildFrom(ResultStatus.BROKER_NOT_EXIST);
}
BrokerMetrics brokerMetrics = brokerService.getBrokerMetricsFromJmx(
clusterId,
brokerId,
KafkaMetricsCollections.BROKER_STATUS_PAGE_METRICS
);
if (ValidateUtils.isNull(brokerMetrics)) {
return Result.buildFrom(ResultStatus.OPERATION_FAILED);
}
Integer underReplicated = brokerMetrics.getSpecifiedMetrics("UnderReplicatedPartitionsValue", Integer.class);
if (ValidateUtils.isNull(underReplicated)) {
return Result.buildFrom(ResultStatus.OPERATION_FAILED);
}
return new Result<>(new ThirdPartBrokerOverviewVO(clusterId, brokerId, underReplicated.equals(0)));
}
@ApiOperation(value = "BrokerRegion信息", notes = "所有集群的")
@RequestMapping(value = "broker-regions", method = RequestMethod.GET)
@ResponseBody
public Result<List<BrokerRegionVO>> getBrokerRegions() {
List<ClusterDO> clusterDOList = clusterService.list();
if (ValidateUtils.isNull(clusterDOList)) {
clusterDOList = new ArrayList<>();
}
List<RegionDO> regionDOList = regionService.listAll();
if (ValidateUtils.isNull(regionDOList)) {
regionDOList = new ArrayList<>();
}
List<BrokerRegionVO> voList = new ArrayList<>();
for (ClusterDO clusterDO: clusterDOList) {
Map<Integer, RegionDO> brokerIdRegionMap = regionService.convert2BrokerIdRegionMap(
regionDOList.stream().filter(elem -> clusterDO.getId().equals(elem.getClusterId())).collect(Collectors.toList())
);
for (Integer brokerId: PhysicalClusterMetadataManager.getBrokerIdList(clusterDO.getId())) {
BrokerRegionVO vo = new BrokerRegionVO();
vo.setClusterId(clusterDO.getId());
vo.setBrokerId(brokerId);
BrokerMetadata metadata = PhysicalClusterMetadataManager.getBrokerMetadata(clusterDO.getId(), brokerId);
if (!ValidateUtils.isNull(metadata)) {
vo.setHostname(metadata.getHost());
}
RegionDO regionDO = brokerIdRegionMap.get(brokerId);
if (!ValidateUtils.isNull(regionDO)) {
vo.setRegionName(regionDO.getName());
}
voList.add(vo);
}
}
return new Result<>(voList);
}
}

View File

@@ -0,0 +1,171 @@
package com.xiaojukeji.kafka.manager.web.api.versionone.thirdpart;
import com.xiaojukeji.kafka.manager.common.bizenum.ConsumeHealthEnum;
import com.xiaojukeji.kafka.manager.common.bizenum.OffsetLocationEnum;
import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
import com.xiaojukeji.kafka.manager.common.constant.Constant;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumeDetailDTO;
import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO;
import com.xiaojukeji.kafka.manager.openapi.common.dto.ConsumeHealthDTO;
import com.xiaojukeji.kafka.manager.openapi.common.dto.OffsetResetDTO;
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO;
import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupDetailVO;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.openapi.ThirdPartService;
import com.xiaojukeji.kafka.manager.openapi.common.vo.ConsumeHealthVO;
import com.xiaojukeji.kafka.manager.service.service.ClusterService;
import com.xiaojukeji.kafka.manager.service.service.ConsumerService;
import com.xiaojukeji.kafka.manager.service.service.gateway.AppService;
import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService;
import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/12
*/
@Api(tags = "开放接口-Consumer相关接口(REST)")
@RestController
@RequestMapping(ApiPrefix.API_V1_THIRD_PART_PREFIX)
public class ThirdPartConsumeController {
private final static Logger LOGGER = LoggerFactory.getLogger(ThirdPartConsumeController.class);
@Autowired
private AppService appService;
@Autowired
private ClusterService clusterService;
@Autowired
private ConsumerService consumerService;
@Autowired
private AuthorityService authorityService;
@Autowired
private ThirdPartService thirdPartService;
@ApiOperation(value = "消费组健康", notes = "消费组是否健康")
@RequestMapping(value = "clusters/consumer-health", method = RequestMethod.POST)
@ResponseBody
public Result<ConsumeHealthVO> checkConsumeHealth(@RequestBody ConsumeHealthDTO dto) {
LOGGER.info("");
if (ValidateUtils.isNull(dto) || !dto.paramLegal()) {
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
}
Result<ConsumeHealthEnum> subEnumResult = null;
for (String topicName: dto.getTopicNameList()) {
subEnumResult = thirdPartService.checkConsumeHealth(
dto.getClusterId(),
topicName,
dto.getConsumerGroup(),
dto.getMaxDelayTime()
);
if (!Constant.SUCCESS.equals(subEnumResult.getCode())) {
return new Result<>(subEnumResult.getCode(), subEnumResult.getMessage());
}
}
if (ValidateUtils.isNull(subEnumResult)) {
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
}
return new Result<>(new ConsumeHealthVO(subEnumResult.getData().getCode()));
}
@ApiOperation(value = "重置消费组", notes = "")
@RequestMapping(value = "consumers/offsets", method = RequestMethod.PUT)
@ResponseBody
public Result<List<Result>> resetOffsets(@RequestBody OffsetResetDTO dto) {
LOGGER.info("rest offset, req:{}.", dto);
if (ValidateUtils.isNull(dto) || !dto.legal()) {
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
}
ClusterDO clusterDO = clusterService.getById(dto.getClusterId());
if (ValidateUtils.isNull(clusterDO)) {
return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
}
// 检查AppID权限
if (!appService.verifyAppIdByPassword(dto.getAppId(), dto.getPassword())) {
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
}
// 检查权限
AuthorityDO authority =
authorityService.getAuthority(dto.getClusterId(), dto.getTopicName(), dto.getAppId());
if (ValidateUtils.isNull(authority) || (authority.getAccess() & 1) <= 0) {
authority = authorityService.getAuthority(dto.getClusterId(), "*", dto.getAppId());
}
if (authority == null || (authority.getAccess() & 1) <= 0) {
return Result.buildFrom(ResultStatus.USER_WITHOUT_AUTHORITY);
}
List<Result> resultList = thirdPartService.resetOffsets(clusterDO, dto);
if (ValidateUtils.isNull(resultList)) {
return Result.buildFrom(ResultStatus.OPERATION_FAILED);
}
for (Result result: resultList) {
if (!Constant.SUCCESS.equals(result.getCode())) {
return Result.buildFrom(ResultStatus.OPERATION_FAILED);
}
}
return new Result<>(resultList);
}
@ApiOperation(value = "查询消费组的消费详情", notes = "")
@RequestMapping(value = "{physicalClusterId}/consumers/{consumerGroup}/topics/{topicName}/consume-details",
method = RequestMethod.GET)
@ResponseBody
public Result<List<ConsumerGroupDetailVO>> getConsumeDetail(@PathVariable Long physicalClusterId,
@PathVariable String consumerGroup,
@PathVariable String topicName,
@RequestParam("location") String location) {
if (ValidateUtils.isNull(location)) {
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
}
ClusterDO clusterDO = clusterService.getById(physicalClusterId);
if (ValidateUtils.isNull(clusterDO)) {
return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
}
location = location.toLowerCase();
OffsetLocationEnum offsetStoreLocation = OffsetLocationEnum.getOffsetStoreLocation(location);
if (ValidateUtils.isNull(offsetStoreLocation)) {
return Result.buildFrom(ResultStatus.CG_LOCATION_ILLEGAL);
}
ConsumerGroupDTO consumeGroupDTO = new ConsumerGroupDTO(
clusterDO.getId(),
consumerGroup,
new ArrayList<>(),
offsetStoreLocation
);
try {
List<ConsumeDetailDTO> consumeDetailDTOList =
consumerService.getConsumeDetail(clusterDO, topicName, consumeGroupDTO);
return new Result<>(
ConsumerModelConverter.convert2ConsumerGroupDetailVO(
topicName,
consumerGroup,
location,
consumeDetailDTOList
)
);
} catch (Exception e) {
LOGGER.error("get consume detail failed, consumerGroup:{}.", consumeGroupDTO, e);
}
return Result.buildFrom(ResultStatus.OPERATION_FAILED);
}
}

View File

@@ -0,0 +1,160 @@
package com.xiaojukeji.kafka.manager.web.api.versionone.thirdpart;
import com.xiaojukeji.kafka.manager.common.bizenum.TopicOffsetChangedEnum;
import com.xiaojukeji.kafka.manager.common.constant.Constant;
import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics;
import com.xiaojukeji.kafka.manager.common.entity.vo.common.RealTimeMetricsVO;
import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadataVO;
import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupVO;
import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicAuthorizedAppVO;
import com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic.TopicRequestTimeDetailVO;
import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata;
import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicOffsetChangedVO;
import com.xiaojukeji.kafka.manager.openapi.common.vo.TopicStatisticMetricsVO;
import com.xiaojukeji.kafka.manager.common.utils.DateUtils;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
import com.xiaojukeji.kafka.manager.service.service.*;
import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
import com.xiaojukeji.kafka.manager.web.converters.CommonModelConverter;
import com.xiaojukeji.kafka.manager.web.converters.ConsumerModelConverter;
import com.xiaojukeji.kafka.manager.web.converters.TopicModelConverter;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.Date;
import java.util.List;
/**
* @author zengqiao
* @date 20/7/24
*/
@Api(tags = "开放接口-Topic相关接口(REST)")
@RestController
@RequestMapping(ApiPrefix.API_V1_THIRD_PART_PREFIX)
public class ThirdPartTopicController {
private final static Logger LOGGER = LoggerFactory.getLogger(ThirdPartTopicController.class);
@Autowired
private TopicService topicService;
@Autowired
private ClusterService clusterService;
@Autowired
private ConsumerService consumerService;
@Autowired
private TopicManagerService topicManagerService;
@ApiOperation(value = "Topic元信息", notes = "LogX调用")
@RequestMapping(value = "clusters/{clusterId}/topics/{topicName}/metadata", method = RequestMethod.GET)
@ResponseBody
public Result<TopicMetadataVO> getTopicMetadata(@PathVariable Long clusterId, @PathVariable String topicName) {
TopicMetadata topicMetadata = PhysicalClusterMetadataManager.getTopicMetadata(clusterId, topicName);
if (ValidateUtils.isNull(topicMetadata)) {
return Result.buildFrom(ResultStatus.TOPIC_NOT_EXIST);
}
TopicMetadataVO vo = new TopicMetadataVO();
vo.setTopicName(topicMetadata.getTopic());
vo.setPartitionNum(topicMetadata.getPartitionNum());
return new Result<>(vo);
}
@ApiOperation(value = "Topic流量统计信息", notes = "")
@RequestMapping(value = "{physicalClusterId}/topics/{topicName}/statistic-metrics", method = RequestMethod.GET)
@ResponseBody
public Result<TopicStatisticMetricsVO> getTopicStatisticMetrics(@PathVariable Long physicalClusterId,
@PathVariable String topicName,
@RequestParam("latest-day") Integer latestDay) {
try {
return new Result<>(new TopicStatisticMetricsVO(topicManagerService.getTopicMaxAvgBytesIn(
physicalClusterId,
topicName,
new Date(DateUtils.getDayStarTime(-1 * latestDay)),
new Date(),
1
)));
} catch (Exception e) {
LOGGER.error("get topic statistic metrics failed, clusterId:{} topicName:{} latestDay:{}."
, physicalClusterId, topicName, latestDay, e);
}
return Result.buildFrom(ResultStatus.MYSQL_ERROR);
}
@ApiOperation(value = "Topic是否有流量", notes = "")
@RequestMapping(value = "{physicalClusterId}/topics/{topicName}/offset-changed", method = RequestMethod.GET)
@ResponseBody
public Result<TopicOffsetChangedVO> checkTopicExpired(@PathVariable Long physicalClusterId,
@PathVariable String topicName,
@RequestParam("latest-time") Long latestTime) {
Result<TopicOffsetChangedEnum> enumResult =
topicService.checkTopicOffsetChanged(physicalClusterId, topicName, latestTime);
if (!Constant.SUCCESS.equals(enumResult.getCode())) {
return new Result<>(enumResult.getCode(), enumResult.getMessage());
}
return new Result<>(new TopicOffsetChangedVO(enumResult.getData().getCode()));
}
@ApiOperation(value = "Topic实时流量信息", notes = "")
@RequestMapping(value = "{physicalClusterId}/topics/{topicName}/metrics", method = RequestMethod.GET)
@ResponseBody
public Result<RealTimeMetricsVO> getTopicMetrics(@PathVariable Long physicalClusterId,
@PathVariable String topicName) {
return new Result<>(CommonModelConverter.convert2RealTimeMetricsVO(
topicService.getTopicMetricsFromJMX(
physicalClusterId,
topicName,
KafkaMetricsCollections.COMMON_DETAIL_METRICS,
true
)
));
}
@ApiOperation(value = "Topic实时请求耗时信息", notes = "")
@RequestMapping(value = "{physicalClusterId}/topics/{topicName}/request-time", method = RequestMethod.GET)
@ResponseBody
public Result<List<TopicRequestTimeDetailVO>> getTopicRequestMetrics(@PathVariable Long physicalClusterId,
@PathVariable String topicName) {
BaseMetrics metrics = topicService.getTopicMetricsFromJMX(
physicalClusterId,
topicName,
KafkaMetricsCollections.TOPIC_REQUEST_TIME_DETAIL_PAGE_METRICS,
false
);
return new Result<>(TopicModelConverter.convert2TopicRequestTimeDetailVOList(metrics));
}
@ApiOperation(value = "查询Topic的消费组列表", notes = "")
@RequestMapping(value = "{physicalClusterId}/topics/{topicName}/consumer-groups", method = RequestMethod.GET)
@ResponseBody
public Result<List<ConsumerGroupVO>> getConsumeDetail(@PathVariable Long physicalClusterId,
@PathVariable String topicName) {
ClusterDO clusterDO = clusterService.getById(physicalClusterId);
if (ValidateUtils.isNull(clusterDO)) {
return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
}
return new Result<>(ConsumerModelConverter.convert2ConsumerGroupVOList(
consumerService.getConsumerGroupList(physicalClusterId, topicName)
));
}
@ApiOperation(value = "Topic应用信息", notes = "")
@RequestMapping(value = "{physicalClusterId}/topics/{topicName}/apps", method = RequestMethod.GET)
@ResponseBody
public Result<List<TopicAuthorizedAppVO>> getTopicAppIds(@PathVariable Long physicalClusterId,
@PathVariable String topicName) {
return new Result<>(TopicModelConverter.convert2TopicAuthorizedAppVOList(
topicManagerService.getTopicAuthorizedApps(physicalClusterId, topicName))
);
}
}

View File

@@ -1,5 +1,6 @@
package com.xiaojukeji.kafka.manager.web.config;
import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
import com.xiaojukeji.kafka.manager.web.inteceptor.PermissionInterceptor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
@@ -31,7 +32,7 @@ public class WebMvcConfig implements WebMvcConfigurer {
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(permissionInterceptor).addPathPatterns("/api/v1/**");
registry.addInterceptor(permissionInterceptor).addPathPatterns(ApiPrefix.API_PREFIX + "**");
}
@Override

View File

@@ -3,7 +3,7 @@ package com.xiaojukeji.kafka.manager.web.inteceptor;
import com.codahale.metrics.Timer;
import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
import com.xiaojukeji.kafka.manager.common.entity.DeprecatedResponseResult;
import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.ao.api.ApiCount;
@@ -118,8 +118,8 @@ public class WebMetricsInterceptor {
ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
String uri = attributes.getRequest().getRequestURI();
if (uri.contains("gateway/api/v1")) {
return DeprecatedResponseResult.failure("api limited");
if (uri.contains(ApiPrefix.GATEWAY_API_V1_PREFIX)) {
return Result.buildFailure("api limited");
}
return new Result<>(ResultStatus.OPERATION_FORBIDDEN);
}