8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

合并Master分支

Browse Source
This commit is contained in:
ZQKC
2023-07-07 13:09:28 +08:00
parent 9f6882cf0d 42195c3180
commit b1aa12bfa5
108 changed files with 2418 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
km-enterprise/km-ha/pom.xml
View File

@@ -5,13 +5,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-ha</artifactId>
<version>${km.revision}</version>
<version>${revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>

2
km-enterprise/km-rebalance/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

8
km-enterprise/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/core/job/ClusterBalanceJobHandler.java
View File

@@ -6,6 +6,7 @@ import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.Enterprise
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.core.service.config.KSConfigUtils;
import com.xiaojukeji.know.streaming.km.rebalance.common.bean.entity.job.ClusterBalanceReassignDetail;
import com.xiaojukeji.know.streaming.km.rebalance.common.bean.entity.job.detail.ClusterBalanceDetailDataGroupByTopic;
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job;
@@ -31,7 +32,6 @@ import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import com.xiaojukeji.know.streaming.km.core.service.config.ConfigUtils;
import com.xiaojukeji.know.streaming.km.core.service.job.JobHandler;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.persistence.mysql.job.JobDAO;
@@ -91,7 +91,7 @@ public class ClusterBalanceJobHandler implements JobHandler {
private TopicService topicService;
@Autowired
private ConfigUtils configUtils;
private KSConfigUtils ksConfigUtils;
@Override
public JobTypeEnum type() {
@@ -118,7 +118,7 @@ public class ClusterBalanceJobHandler implements JobHandler {
}
//获取任务计划
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(dto.getClusterId(), configUtils.getClusterBalanceIgnoredTopicsTimeSecond());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(dto.getClusterId(), ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond());
BalanceParameter balanceParameter = ClusterBalanceConverter.convert2BalanceParameter(dto, brokers, brokerSpecMap, clusterPhy, esAddress, esPassword, topicNames);
try {
ExecutionRebalance executionRebalance = new ExecutionRebalance();
@@ -203,7 +203,7 @@ public class ClusterBalanceJobHandler implements JobHandler {
List<Broker> brokers = brokerService.listAllBrokersFromDB(clusterPhy.getId());
Map<Integer, BrokerSpec> brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhy.getId());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(job.getClusterId(), configUtils.getClusterBalanceIgnoredTopicsTimeSecond());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(job.getClusterId(), ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond());
JobClusterBalanceContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobClusterBalanceContent.class);
BalanceParameter balanceParameter = ClusterBalanceConverter.convert2BalanceParameter(dto, brokers, brokerSpecMap, clusterPhy, esAddress, esPassword, topicNames);
ExecutionRebalance executionRebalance = new ExecutionRebalance();

10
km-enterprise/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/core/service/impl/ClusterBalanceJobServiceImpl.java
View File

@@ -22,6 +22,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
import com.xiaojukeji.know.streaming.km.core.service.config.KSConfigUtils;
import com.xiaojukeji.know.streaming.km.rebalance.common.bean.entity.ClusterBalanceInterval;
import com.xiaojukeji.know.streaming.km.rebalance.common.bean.entity.job.detail.ClusterBalancePlanDetail;
import com.xiaojukeji.know.streaming.km.rebalance.common.bean.po.ClusterBalanceJobConfigPO;
@@ -37,8 +38,6 @@ import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import com.xiaojukeji.know.streaming.km.core.service.config.ConfigUtils;
import com.xiaojukeji.know.streaming.km.core.service.partition.OpPartitionService;
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignService;
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignStrategyService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
@@ -96,7 +95,7 @@ public class ClusterBalanceJobServiceImpl implements ClusterBalanceJobService {
private TopicService topicService;
@Autowired
private ConfigUtils configUtils;
private KSConfigUtils ksConfigUtils;
@Autowired
private ReassignService reassignService;
@@ -104,9 +103,6 @@ public class ClusterBalanceJobServiceImpl implements ClusterBalanceJobService {
@Autowired
private ReassignStrategyService reassignStrategyService;
@Autowired
private OpPartitionService opPartitionService;
@Override
public Result<Void> deleteByJobId(Long jobId, String operator) {
if (jobId == null) {
@@ -304,7 +300,7 @@ public class ClusterBalanceJobServiceImpl implements ClusterBalanceJobService {
Map<Integer, Broker> brokerMap = brokerService.listAllBrokersFromDB(clusterBalanceJobPO.getClusterId()).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
//更新平衡任务状态信息
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhy.getId(), configUtils.getClusterBalanceIgnoredTopicsTimeSecond());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhy.getId(), ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond());
Map<Integer, BrokerBalanceState> brokerBalanceStateMap = ExecutionRebalance
.getBrokerResourcesBalanceState(ClusterBalanceConverter.convert2BalanceParameter(clusterBalanceJobPO, brokerMap, brokerSpecMap, clusterPhy, esAddress, esPassword, topicNames));
List<ClusterBalancePlanDetail> oldDetails = ConvertUtil.str2ObjArrayByJson(clusterBalanceJobPO.getBrokerBalanceDetail(), ClusterBalancePlanDetail.class);

12
km-enterprise/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/core/service/impl/ClusterBalanceServiceImpl.java
View File

@@ -11,7 +11,7 @@ import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import com.xiaojukeji.know.streaming.km.core.service.config.ConfigUtils;
import com.xiaojukeji.know.streaming.km.core.service.config.KSConfigUtils;
import com.xiaojukeji.know.streaming.km.core.service.job.JobService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.BrokerMetricVersionItems;
@@ -96,7 +96,7 @@ public class ClusterBalanceServiceImpl implements ClusterBalanceService {
private TopicService topicService;
@Autowired
private ConfigUtils configUtils;
private KSConfigUtils ksConfigUtils;
private final Cache<Long, Result<ClusterBalanceItemState>> balanceStateCache = Caffeine.newBuilder()
.expireAfterWrite(150, TimeUnit.SECONDS)
@@ -134,7 +134,7 @@ public class ClusterBalanceServiceImpl implements ClusterBalanceService {
} catch (ParseException e) {
logger.error("method=state||clusterId:{}||errMsg=exception", clusterPhyId, e);
}
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond());
clusterBalanceStateVO.setEnable(configPOResult.getData().getStatus() == 1);
Map<Resource, Double> resourceDoubleMap;
@@ -190,7 +190,7 @@ public class ClusterBalanceServiceImpl implements ClusterBalanceService {
Map<Integer, BrokerBalanceState> brokerBalanceStateMap = new HashMap<>();
if (configPOResult.hasData()) {
try {
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond());
brokerBalanceStateMap = ExecutionRebalance
.getBrokerResourcesBalanceState(ClusterBalanceConverter.convert2BalanceParameter(configPOResult.getData(), brokerMap, brokerSpecMap, clusterPhy, esAddress, esPassword, topicNames));
} catch (Exception e) {
@@ -268,7 +268,7 @@ public class ClusterBalanceServiceImpl implements ClusterBalanceService {
// Topic信息
List<String> recentTopicNameList = topicService.listRecentUpdateTopicNamesFromDB(
clusterPhyId,
configUtils.getClusterBalanceIgnoredTopicsTimeSecond()
ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond()
);
ClusterBalanceItemState balanceState = new ClusterBalanceItemState();
@@ -382,7 +382,7 @@ public class ClusterBalanceServiceImpl implements ClusterBalanceService {
//获取任务计划
Map<Integer, Broker> brokerMap = allBrokers.stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond());
List<String> topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, ksConfigUtils.getClusterBalanceIgnoredTopicsTimeSecond());
BalanceParameter balanceParameter = ClusterBalanceConverter.convert2BalanceParameter(clusterBalancePreviewDTO, brokerMap, brokerSpecMap, clusterPhy, esAddress, esPassword, topicNames);
ExecutionRebalance executionRebalance = new ExecutionRebalance();
try {

4
km-enterprise/km-testing/pom.xml
View File

@@ -5,13 +5,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-testing</artifactId>
<version>${km.revision}</version>
<version>${revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>