v2.1版本更新

2026-01-01 17:52:11 +08:00 · 2020-12-19 00:27:16 +08:00
parent 3fea5c9c8c
commit 49280a8617
75 changed files with 1098 additions and 148 deletions
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java
@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
 import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
 import com.xiaojukeji.kafka.manager.common.entity.Result;
+import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.TopicConnectionDO;
 import com.xiaojukeji.kafka.manager.common.utils.JsonUtils;
 import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
 import com.xiaojukeji.kafka.manager.service.service.gateway.TopicConnectionService;
@@ -15,6 +16,8 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;

+import java.util.List;
+
 /**
 * @author zengqiao
 * @date 20/7/6
@@ -35,15 +38,22 @@ public class GatewayHeartbeatController {
    public Result receiveTopicConnections(@RequestParam("clusterId") Long clusterId,
                                          @RequestParam("brokerId") Integer brokerId,
                                          @RequestBody JSONObject jsonObject) {
-        try {
-            if (ValidateUtils.isNull(jsonObject) || jsonObject.isEmpty()) {
-                return Result.buildSuc();
-            }
-            topicConnectionService.batchAdd(JsonUtils.parseTopicConnections(clusterId, jsonObject));
+        if (ValidateUtils.isNull(jsonObject) || jsonObject.isEmpty()) {
+            LOGGER.info("class=GatewayHeartbeatController||method=receiveTopicConnections||clusterId={}||brokerId={}||msg=connections empty!", clusterId, brokerId);
            return Result.buildSuc();
-        } catch (Exception e) {
-            LOGGER.error("receive topic connections failed, clusterId:{} brokerId:{} req:{}", clusterId, brokerId, jsonObject, e);
        }
-        return Result.buildFailure("fail");
+
+        LOGGER.info("class=GatewayHeartbeatController||method=receiveTopicConnections||clusterId={}||brokerId={}||size={}||msg=receive connections", clusterId, brokerId, jsonObject.size());
+
+        List<TopicConnectionDO> doList = null;
+        try {
+            doList = JsonUtils.parseTopicConnections(clusterId, jsonObject);
+        } catch (Exception e) {
+            LOGGER.error("class=GatewayHeartbeatController||method=receiveTopicConnections||clusterId={}||brokerId={}||msg=parse data failed||exception={}", clusterId, brokerId, e.getMessage());
+            return Result.buildFailure("fail");
+        }
+
+        topicConnectionService.batchAdd(doList);
+        return Result.buildSuc();
    }
 }
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayServiceDiscoveryController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayServiceDiscoveryController.java
@@ -14,6 +14,8 @@ import com.xiaojukeji.kafka.manager.service.service.gateway.GatewayConfigService
 import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;

@@ -29,6 +31,9 @@ import java.util.Map;
@RestController
@RequestMapping(ApiPrefix.GATEWAY_API_V1_PREFIX)
 public class GatewayServiceDiscoveryController {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(GatewayHeartbeatController.class);
+
    @Autowired
    private GatewayConfigService gatewayConfigService;

@@ -38,6 +43,7 @@ public class GatewayServiceDiscoveryController {
    @ResponseBody
    public String getKafkaBootstrapServer(@RequestParam("clusterId") Long clusterId) {
        if (ValidateUtils.isNull(clusterId)) {
+            LOGGER.warn("class=GatewayServiceDiscoveryController||method=getKafkaBootstrapServer||msg=param clusterId is null!");
            return "";
        }
        GatewayConfigDO configDO = gatewayConfigService.getByTypeAndName(
@@ -45,6 +51,7 @@ public class GatewayServiceDiscoveryController {
                String.valueOf(clusterId)
        );
        if (ValidateUtils.isNull(configDO)) {
+            LOGGER.info("class=GatewayServiceDiscoveryController||method=getKafkaBootstrapServer||msg=configDO is null!");
            return "";
        }
        return configDO.getValue();
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAccountController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAccountController.java
@@ -9,8 +9,11 @@ import com.xiaojukeji.kafka.manager.common.entity.vo.common.AccountSummaryVO;
 import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
 import com.xiaojukeji.kafka.manager.common.utils.SpringTool;
 import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix;
+import com.xiaojukeji.kafka.manager.web.api.versionone.gateway.GatewayHeartbeatController;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;

@@ -25,6 +28,9 @@ import java.util.List;
@RestController
@RequestMapping(ApiPrefix.API_V1_NORMAL_PREFIX)
 public class NormalAccountController {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(NormalAccountController.class);
+
    @Autowired
    private AccountService accountService;

@@ -34,6 +40,8 @@ public class NormalAccountController {
    public Result<List<AccountSummaryVO>> searchOnJobStaffByKeyWord(@RequestParam("keyWord") String keyWord) {
        List<EnterpriseStaff> staffList = accountService.searchAccountByPrefix(keyWord);
        if (ValidateUtils.isEmptyList(staffList)) {
+            LOGGER.info("class=NormalAccountController||method=searchOnJobStaffByKeyWord||keyWord={}||msg=staffList is empty!"
+                    ,keyWord);
            return new Result<>();
        }
        List<AccountSummaryVO> voList = new ArrayList<>();
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAppController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAppController.java
@@ -1,5 +1,6 @@
 package com.xiaojukeji.kafka.manager.web.api.versionone.normal;

+import com.xiaojukeji.kafka.manager.account.AccountService;
 import com.xiaojukeji.kafka.manager.common.annotations.ApiLevel;
 import com.xiaojukeji.kafka.manager.common.bizenum.TopicAuthorityEnum;
 import com.xiaojukeji.kafka.manager.common.constant.ApiLevelContent;
@@ -8,6 +9,7 @@ import com.xiaojukeji.kafka.manager.common.entity.Result;
 import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
 import com.xiaojukeji.kafka.manager.common.entity.ao.AppTopicDTO;
 import com.xiaojukeji.kafka.manager.common.entity.dto.normal.AppDTO;
+import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.normal.QuotaVO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.normal.app.AppTopicAuthorityVO;
 import com.xiaojukeji.kafka.manager.common.entity.vo.normal.app.AppTopicVO;
@@ -45,6 +47,9 @@ public class NormalAppController {
    @Autowired
    private AppService appService;

+    @Autowired
+    private AccountService accountService;
+
    @Autowired
    private QuotaService quotaService;

@@ -71,9 +76,16 @@ public class NormalAppController {
    @RequestMapping(value = "apps/{appId}/basic-info", method = RequestMethod.GET)
    @ResponseBody
    public Result<AppVO> getAppBasicInfo(@PathVariable String appId) {
-        return new Result<>(AppConverter.convert2AppVO(
-                appService.getByAppId(appId))
-        );
+        if (accountService.isAdminOrderHandler(SpringTool.getUserName())) {
+            return new Result<>(AppConverter.convert2AppVO(appService.getByAppId(appId)));
+        }
+
+        AppDO appDO = appService.getAppByUserAndId(appId, SpringTool.getUserName());
+        if (appDO == null) {
+            return Result.buildFrom(ResultStatus.USER_WITHOUT_AUTHORITY);
+        }
+
+        return new Result<>(AppConverter.convert2AppVO(appDO));
    }

    @ApiOperation(value = "App修改", notes = "")
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalConsumerController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalConsumerController.java
@@ -79,6 +79,7 @@ public class NormalConsumerController {
            @RequestParam("location") String location,
            @RequestParam(value = "isPhysicalClusterId", required = false) Boolean isPhysicalClusterId) {
        if (ValidateUtils.isNull(location)) {
+
            return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
        }
        Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(clusterId, isPhysicalClusterId);
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalTopicController.java
@@ -4,6 +4,7 @@ import com.xiaojukeji.kafka.manager.common.constant.Constant;
 import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections;
 import com.xiaojukeji.kafka.manager.common.entity.Result;
 import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
+import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicConnection;
 import com.xiaojukeji.kafka.manager.common.entity.ao.topic.TopicPartitionDTO;
 import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicDataSampleDTO;
 import com.xiaojukeji.kafka.manager.common.entity.metrics.BaseMetrics;
@@ -14,6 +15,7 @@ import com.xiaojukeji.kafka.manager.common.utils.SpringTool;
 import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
 import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
 import com.xiaojukeji.kafka.manager.common.entity.pojo.KafkaBillDO;
+import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxAttributeEnum;
 import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager;
 import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
 import com.xiaojukeji.kafka.manager.service.service.*;
@@ -134,18 +136,27 @@ public class NormalTopicController {
    public Result<List<TopicRequestTimeDetailVO>> getTopicRequestMetrics(
            @PathVariable Long clusterId,
            @PathVariable String topicName,
-            @RequestParam(value = "isPhysicalClusterId", required = false) Boolean isPhysicalClusterId) {
+            @RequestParam(value = "isPhysicalClusterId", required = false) Boolean isPhysicalClusterId,
+            @RequestParam(value = "percentile", required = false, defaultValue = "75thPercentile") String percentile) {
        Long physicalClusterId = logicalClusterMetadataManager.getPhysicalClusterId(clusterId, isPhysicalClusterId);
        if (ValidateUtils.isNull(physicalClusterId)) {
            return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
        }
+
+        Boolean isPercentileLegal = Arrays.stream(JmxAttributeEnum.PERCENTILE_ATTRIBUTE.getAttribute())
+                .anyMatch(percentile::equals);
+
+        if (!isPercentileLegal) {
+            return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
+        }
+
        BaseMetrics metrics = topicService.getTopicMetricsFromJMX(
                physicalClusterId,
                topicName,
                KafkaMetricsCollections.TOPIC_REQUEST_TIME_DETAIL_PAGE_METRICS,
                false
        );
-        return new Result<>(TopicModelConverter.convert2TopicRequestTimeDetailVOList(metrics));
+        return new Result<>(TopicModelConverter.convert2TopicRequestTimeDetailVOList(metrics, percentile));
    }

    @ApiOperation(value = "Topic历史请求耗时信息", notes = "")
@@ -184,14 +195,26 @@ public class NormalTopicController {
            return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
        }

-        return new Result<>(TopicModelConverter.convert2TopicConnectionVOList(
-                connectionService.getByTopicName(
-                        physicalClusterId,
-                        topicName,
-                        new Date(System.currentTimeMillis() - Constant.TOPIC_CONNECTION_LATEST_TIME_MS),
-                        new Date()
-                )
-        ));
+        List<TopicConnection> connections;
+
+        if (ValidateUtils.isBlank(appId)) {
+            connections = connectionService.getByTopicName(
+                    physicalClusterId,
+                    topicName,
+                    new Date(System.currentTimeMillis() - Constant.TOPIC_CONNECTION_LATEST_TIME_MS),
+                    new Date()
+            );
+        } else {
+            connections = connectionService.getByTopicName(
+                    physicalClusterId,
+                    topicName,
+                    appId,
+                    new Date(System.currentTimeMillis() - Constant.TOPIC_CONNECTION_LATEST_TIME_MS),
+                    new Date()
+            );
+        }
+
+        return new Result<>(TopicModelConverter.convert2TopicConnectionVOList(connections));
    }

    @ApiOperation(value = "Topic分区信息", notes = "")
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdTopicController.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdTopicController.java
@@ -70,6 +70,7 @@ public class RdTopicController {
        RdTopicBasicVO vo = new RdTopicBasicVO();
        CopyUtils.copyProperties(vo, result.getData());
        vo.setProperties(result.getData().getProperties());
+        vo.setRegionNameList(result.getData().getRegionNameList());
        return new Result<>(vo);
    }
 }
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/config/RestTemplateConfig.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/config/RestTemplateConfig.java
@@ -0,0 +1,29 @@
+package com.xiaojukeji.kafka.manager.web.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.ClientHttpRequestFactory;
+import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.web.client.RestTemplate;
+
+/**
+ * author: mrazkonglingxu
+ * Date: 2020/12/2
+ * Time: 10:48 上午
+ */
+@Configuration
+public class RestTemplateConfig {
+
+    @Bean
+    public ClientHttpRequestFactory simpleClientHttpRequestFactory() {
+        SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
+        factory.setConnectTimeout(5000);
+        factory.setReadTimeout(5000);
+        return factory;
+    }
+
+    @Bean
+    public RestTemplate restTemplate(ClientHttpRequestFactory factory) {
+        return new RestTemplate(factory);
+    }
+}
--- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicModelConverter.java
+++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicModelConverter.java
@@ -40,6 +40,7 @@ public class TopicModelConverter {
        if (!ValidateUtils.isNull(clusterDO)) {
            vo.setBootstrapServers(clusterDO.getBootstrapServers());
        }
+        vo.setRegionNameList(dto.getRegionNameList());
        return vo;
    }

@@ -107,6 +108,54 @@ public class TopicModelConverter {
        return Arrays.asList(produceVO, fetchVO);
    }

+    public static List<TopicRequestTimeDetailVO> convert2TopicRequestTimeDetailVOList(BaseMetrics metrics, String percentile) {
+        if (ValidateUtils.isNull(metrics)) {
+            return new ArrayList<>();
+        }
+        TopicRequestTimeDetailVO produceVO = new TopicRequestTimeDetailVO();
+        produceVO.setRequestTimeType("RequestProduceTime");
+        fillTopicProduceTime(produceVO, metrics, percentile);
+
+        TopicRequestTimeDetailVO fetchVO = new TopicRequestTimeDetailVO();
+        fetchVO.setRequestTimeType("RequestFetchTime");
+        fillTopicFetchTime(fetchVO, metrics, percentile);
+
+        TopicMetrics topicMetrics = (TopicMetrics) metrics;
+        if (!ValidateUtils.isEmptyList(topicMetrics.getBrokerMetricsList())) {
+            List<TopicBrokerRequestTimeVO> brokerProduceTimeList = new ArrayList<>();
+            List<TopicBrokerRequestTimeVO> brokerFetchTimeList = new ArrayList<>();
+            topicMetrics.getBrokerMetricsList().forEach(brokerMetrics -> {
+                TopicBrokerRequestTimeVO topicBrokerProduceReq = new TopicBrokerRequestTimeVO();
+                topicBrokerProduceReq.setClusterId(brokerMetrics.getClusterId());
+                topicBrokerProduceReq.setBrokerId(brokerMetrics.getBrokerId());
+
+                TopicRequestTimeDetailVO brokerProduceVO = new TopicRequestTimeDetailVO();
+                brokerProduceVO.setRequestTimeType("BrokerRequestProduceTime");
+                fillTopicProduceTime(brokerProduceVO, brokerMetrics, percentile);
+
+                topicBrokerProduceReq.setBrokerRequestTime(brokerProduceVO);
+
+                TopicBrokerRequestTimeVO topicBrokerFetchReq = new TopicBrokerRequestTimeVO();
+                topicBrokerFetchReq.setClusterId(brokerMetrics.getClusterId());
+                topicBrokerFetchReq.setBrokerId(brokerMetrics.getBrokerId());
+
+                TopicRequestTimeDetailVO brokerFetchVO = new TopicRequestTimeDetailVO();
+                brokerProduceVO.setRequestTimeType("BrokerRequestFetchTime");
+                fillTopicFetchTime(brokerFetchVO, brokerMetrics, percentile);
+
+                topicBrokerFetchReq.setBrokerRequestTime(brokerFetchVO);
+
+                brokerProduceTimeList.add(topicBrokerProduceReq);
+                brokerFetchTimeList.add(topicBrokerFetchReq);
+            });
+
+            produceVO.setBrokerRequestTimeList(brokerProduceTimeList);
+            fetchVO.setBrokerRequestTimeList(brokerFetchTimeList);
+        }
+
+        return Arrays.asList(produceVO, fetchVO);
+    }
+
    public static List<TopicConnectionVO> convert2TopicConnectionVOList(List<TopicConnection> connectionDTOList) {
        if (ValidateUtils.isNull(connectionDTOList)) {
            return new ArrayList<>();
@@ -224,4 +273,24 @@ public class TopicModelConverter {
        CopyUtils.copyProperties(topicBusinessInfoVO,topicBusinessInfo);
        return topicBusinessInfoVO;
    }
+
+    private static void fillTopicProduceTime(TopicRequestTimeDetailVO produceVO, BaseMetrics metrics, String thPercentile) {
+        produceVO.setRequestQueueTimeMs(metrics.getSpecifiedMetrics("ProduceRequestQueueTimeMs" + thPercentile));
+        produceVO.setResponseQueueTimeMs(metrics.getSpecifiedMetrics("ProduceResponseQueueTimeMs" + thPercentile));
+        produceVO.setResponseSendTimeMs(metrics.getSpecifiedMetrics("ProduceResponseSendTimeMs" + thPercentile));
+        produceVO.setLocalTimeMs(metrics.getSpecifiedMetrics("ProduceLocalTimeMs" + thPercentile));
+        produceVO.setThrottleTimeMs(metrics.getSpecifiedMetrics("ProduceThrottleTimeMs" + thPercentile));
+        produceVO.setRemoteTimeMs(metrics.getSpecifiedMetrics("ProduceRemoteTimeMs" + thPercentile));
+        produceVO.setTotalTimeMs(metrics.getSpecifiedMetrics("ProduceTotalTimeMs"  + thPercentile));
+    }
+
+    private static void fillTopicFetchTime(TopicRequestTimeDetailVO fetchVO, BaseMetrics metrics, String thPercentile) {
+        fetchVO.setRequestQueueTimeMs(metrics.getSpecifiedMetrics("FetchConsumerRequestQueueTimeMs" + thPercentile));
+        fetchVO.setResponseQueueTimeMs(metrics.getSpecifiedMetrics("FetchConsumerResponseQueueTimeMs" + thPercentile));
+        fetchVO.setResponseSendTimeMs(metrics.getSpecifiedMetrics("FetchConsumerResponseSendTimeMs" + thPercentile));
+        fetchVO.setLocalTimeMs(metrics.getSpecifiedMetrics("FetchConsumerLocalTimeMs" + thPercentile));
+        fetchVO.setThrottleTimeMs(metrics.getSpecifiedMetrics("FetchConsumerThrottleTimeMs" + thPercentile));
+        fetchVO.setRemoteTimeMs(metrics.getSpecifiedMetrics("FetchConsumerRemoteTimeMs" + thPercentile));
+        fetchVO.setTotalTimeMs(metrics.getSpecifiedMetrics("FetchConsumerTotalTimeMs" + thPercentile));
+    }
 }