From c56d8cfb0f72cf1e8953f67ee310b08eacd8bacf Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 23 Feb 2023 11:56:46 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0rebalance=20/=20testing=20/?= =?UTF-8?q?=20license=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/MultiClusterPhyManagerImpl.java | 5 +- km-common/pom.xml | 6 + .../bean/dto/ClusterBalanceIntervalDTO.java | 26 + .../bean/dto/ClusterBalanceOverviewDTO.java | 20 + .../bean/dto/ClusterBalancePreviewDTO.java | 43 ++ .../bean/dto/ClusterBalanceStrategyDTO.java | 66 ++ .../bean/entity/ClusterBalanceInterval.java | 27 + .../bean/entity/ClusterBalanceItemState.java | 41 ++ .../entity/job/ClusterBalanceJobConfig.java | 91 +++ .../entity/job/ClusterBalanceReassign.java | 64 ++ .../job/ClusterBalanceReassignDetail.java | 36 ++ .../job/ClusterBalanceReassignExtendData.java | 47 ++ .../job/content/JobClusterBalanceContent.java | 43 ++ .../AbstractClusterBalanceDetailData.java | 79 +++ ...sterBalanceDetailDataGroupByPartition.java | 17 + .../ClusterBalanceDetailDataGroupByTopic.java | 22 + .../job/detail/ClusterBalancePlanDetail.java | 76 +++ .../bean/po/ClusterBalanceJobConfigPO.java | 85 +++ .../bean/po/ClusterBalanceJobPO.java | 125 ++++ .../bean/po/ClusterBalanceReassignPO.java | 80 +++ .../bean/vo/ClusterBalanceHistorySubVO.java | 24 + .../bean/vo/ClusterBalanceHistoryVO.java | 34 + .../bean/vo/ClusterBalanceIntervalVO.java | 20 + .../bean/vo/ClusterBalanceJobConfigVO.java | 55 ++ .../bean/vo/ClusterBalanceOverviewSubVO.java | 32 + .../bean/vo/ClusterBalanceOverviewVO.java | 37 ++ .../bean/vo/ClusterBalancePlanDetailVO.java | 64 ++ .../bean/vo/ClusterBalancePlanVO.java | 49 ++ .../bean/vo/ClusterBalanceStateSubVO.java | 32 + .../bean/vo/ClusterBalanceStateVO.java | 32 + .../converter/ClusterBalanceConverter.java | 476 ++++++++++++++ .../ClusterBalanceReassignConverter.java | 218 +++++++ .../enums/ClusterBalanceStateEnum.java | 31 + .../enums/ClusterBalanceTypeEnum.java | 28 + .../enterprise/rebalance/package-info.java | 9 + .../enums/operaterecord/ModuleEnum.java | 4 + .../job/ClusterBalanceJobHandler.java | 355 +++++++++++ .../enterprise/rebalance/package-info.java | 9 + .../ClusterBalanceJobConfigService.java | 23 + .../service/ClusterBalanceJobService.java | 93 +++ .../ClusterBalanceReassignService.java | 72 +++ .../service/ClusterBalanceService.java | 82 +++ .../ClusterBalanceJobConfigServiceImpl.java | 79 +++ .../impl/ClusterBalanceJobServiceImpl.java | 458 ++++++++++++++ .../ClusterBalanceReassignServiceImpl.java | 514 +++++++++++++++ .../impl/ClusterBalanceServiceImpl.java | 481 ++++++++++++++ .../impl/ClusterMetricServiceImpl.java | 35 + .../km/core/service/config/ConfigUtils.java | 4 + .../metrics/ClusterMetricVersionItems.java | 31 + km-enterprise/km-testing/pom.xml | 33 + .../km/testing/KmTestingApplication.java | 12 + .../testing/biz/KafkaClientTestManager.java | 30 + .../biz/impl/KafkaClientTestManagerImpl.java | 596 ++++++++++++++++++ .../common/bean/dto/KafkaConsumerDTO.java | 47 ++ .../bean/dto/KafkaConsumerFilterDTO.java | 34 + .../bean/dto/KafkaConsumerStartFromDTO.java | 40 ++ .../common/bean/dto/KafkaProducerDTO.java | 46 ++ .../km/testing/common/bean/vo/BaseTestVO.java | 22 + .../common/bean/vo/TestConsumerVO.java | 33 + .../bean/vo/TestPartitionConsumedVO.java | 31 + .../common/bean/vo/TestProducerVO.java | 33 + .../common/enums/KafkaConsumerFilterEnum.java | 35 + .../enums/KafkaConsumerStartFromEnum.java | 35 + .../km/testing/common/package-info.java | 7 + .../testing/rest/KafkaClientController.java | 46 ++ km-extends/km-license/pom.xml | 40 ++ .../km/license/LicenseInterceptor.java | 69 ++ .../km/license/LicenseWebConfig.java | 24 + .../streaming/km/license/bean/KmLicense.java | 11 + .../km/license/bean/KmLicenseUsageDetail.java | 24 + .../km/license/bean/LicenseInfo.java | 34 + .../km/license/bean/LicenseResult.java | 12 + .../km/license/bean/LicenseUsage.java | 24 + .../license/controller/LicenseController.java | 27 + .../km/license/service/LicenseService.java | 18 + .../service/impl/LicenseServiceImpl.java | 253 ++++++++ km-extends/km-rebalance/pom.xml | 67 ++ .../km/rebalance/KafkaRebalanceMain.java | 139 ++++ .../OptimizationFailureException.java | 15 + .../executor/ExecutionRebalance.java | 78 +++ .../executor/common/BalanceActionHistory.java | 76 +++ .../executor/common/BalanceDetailed.java | 173 +++++ .../executor/common/BalanceGoal.java | 20 + .../executor/common/BalanceOverview.java | 102 +++ .../executor/common/BalanceParameter.java | 199 ++++++ .../executor/common/BalanceTask.java | 43 ++ .../executor/common/BalanceThreshold.java | 41 ++ .../executor/common/BrokerBalanceState.java | 144 +++++ .../km/rebalance/executor/common/HostEnv.java | 76 +++ .../executor/common/OptimizerResult.java | 218 +++++++ .../executor/common/TopicChangeHistory.java | 78 +++ .../streaming/km/rebalance/metric/Metric.java | 51 ++ .../km/rebalance/metric/MetricStore.java | 9 + .../km/rebalance/metric/Metrics.java | 46 ++ .../ElasticsearchMetricStore.java | 109 ++++ .../streaming/km/rebalance/model/Broker.java | 222 +++++++ .../km/rebalance/model/Capacity.java | 36 ++ .../km/rebalance/model/ClusterModel.java | 236 +++++++ .../streaming/km/rebalance/model/Load.java | 42 ++ .../km/rebalance/model/Partition.java | 148 +++++ .../streaming/km/rebalance/model/Rack.java | 67 ++ .../streaming/km/rebalance/model/Replica.java | 129 ++++ .../rebalance/model/ReplicaPlacementInfo.java | 48 ++ .../km/rebalance/model/Resource.java | 29 + .../km/rebalance/model/Supplier.java | 112 ++++ .../rebalance/optimizer/ActionAcceptance.java | 5 + .../km/rebalance/optimizer/ActionType.java | 18 + .../km/rebalance/optimizer/AnalyzerUtils.java | 73 +++ .../rebalance/optimizer/BalancingAction.java | 40 ++ .../optimizer/ExecutionProposal.java | 72 +++ .../km/rebalance/optimizer/GoalOptimizer.java | 48 ++ .../optimizer/OptimizationOptions.java | 60 ++ .../optimizer/goals/AbstractGoal.java | 129 ++++ .../optimizer/goals/DiskDistributionGoal.java | 31 + .../km/rebalance/optimizer/goals/Goal.java | 17 + .../goals/NetworkInboundDistributionGoal.java | 30 + .../NetworkOutboundDistributionGoal.java | 22 + .../goals/ResourceDistributionGoal.java | 227 +++++++ .../goals/TopicLeadersDistributionGoal.java | 222 +++++++ .../goals/TopicReplicaDistributionGoal.java | 287 +++++++++ .../km/rebalance/utils/CommandLineUtils.java | 21 + .../km/rebalance/utils/GoalUtils.java | 59 ++ .../km/rebalance/utils/MetadataUtils.java | 92 +++ ...treaming.km.rebalance.optimizer.goals.Goal | 5 + .../src/main/resources/MetricsQuery.json | 75 +++ .../know/streaming/km/rebalance/Test.java | 93 +++ .../src/test/resources/log4j.properties | 21 + .../rebalance/ClusterBalanceJobConfigDao.java | 21 + .../rebalance/ClusterBalanceJobDao.java | 23 + .../rebalance/ClusterBalanceReassignDao.java | 24 + .../mybatis/ClusterBalanceJobMapper.xml | 84 +++ .../mybatis/ClusterBalanceReassignMapper.xml | 54 ++ km-rest/pom.xml | 10 + .../rebalance/ClusterBalanceController.java | 88 +++ km-rest/src/main/resources/application.yml | 16 +- .../rebalance/job/ClusterBalanceJobTask.java | 54 ++ pom.xml | 5 +- 137 files changed, 10772 insertions(+), 3 deletions(-) create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceIntervalDTO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceOverviewDTO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalancePreviewDTO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceStrategyDTO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceInterval.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceItemState.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceJobConfig.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassign.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignDetail.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignExtendData.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/content/JobClusterBalanceContent.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/AbstractClusterBalanceDetailData.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByPartition.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByTopic.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalancePlanDetail.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobConfigPO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobPO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceReassignPO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistorySubVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistoryVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceIntervalVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceJobConfigVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewSubVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanDetailVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateSubVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateVO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceConverter.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceReassignConverter.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceStateEnum.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceTypeEnum.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/package-info.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/job/ClusterBalanceJobHandler.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/package-info.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobConfigService.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobService.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceReassignService.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceService.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobConfigServiceImpl.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobServiceImpl.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceReassignServiceImpl.java create mode 100644 km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceServiceImpl.java create mode 100644 km-enterprise/km-testing/pom.xml create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/KmTestingApplication.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/KafkaClientTestManager.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/impl/KafkaClientTestManagerImpl.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerDTO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerFilterDTO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerStartFromDTO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaProducerDTO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/BaseTestVO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestConsumerVO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestPartitionConsumedVO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestProducerVO.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerFilterEnum.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerStartFromEnum.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/package-info.java create mode 100644 km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/rest/KafkaClientController.java create mode 100644 km-extends/km-license/pom.xml create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseInterceptor.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseWebConfig.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicense.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicenseUsageDetail.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseInfo.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseResult.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseUsage.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/controller/LicenseController.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/LicenseService.java create mode 100644 km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/impl/LicenseServiceImpl.java create mode 100644 km-extends/km-rebalance/pom.xml create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/KafkaRebalanceMain.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/exception/OptimizationFailureException.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/ExecutionRebalance.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceActionHistory.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceDetailed.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceOverview.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceParameter.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceTask.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceThreshold.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BrokerBalanceState.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/HostEnv.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/OptimizerResult.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/TopicChangeHistory.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metric.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/MetricStore.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metrics.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/elasticsearch/ElasticsearchMetricStore.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Broker.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Capacity.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ClusterModel.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Load.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Partition.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Rack.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Replica.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ReplicaPlacementInfo.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Resource.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Supplier.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionAcceptance.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionType.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/AnalyzerUtils.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/BalancingAction.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ExecutionProposal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/GoalOptimizer.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/OptimizationOptions.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/AbstractGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/DiskDistributionGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/Goal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkInboundDistributionGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkOutboundDistributionGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/ResourceDistributionGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicLeadersDistributionGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicReplicaDistributionGoal.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/CommandLineUtils.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/GoalUtils.java create mode 100644 km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/MetadataUtils.java create mode 100644 km-extends/km-rebalance/src/main/resources/META-INF/services/com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal create mode 100644 km-extends/km-rebalance/src/main/resources/MetricsQuery.json create mode 100644 km-extends/km-rebalance/src/test/java/com/xiaojukeji/know/streaming/km/rebalance/Test.java create mode 100644 km-extends/km-rebalance/src/test/resources/log4j.properties create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobConfigDao.java create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobDao.java create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceReassignDao.java create mode 100644 km-persistence/src/main/resources/mybatis/ClusterBalanceJobMapper.xml create mode 100644 km-persistence/src/main/resources/mybatis/ClusterBalanceReassignMapper.xml create mode 100644 km-rest/src/main/java/com/xiaojukeji/know/streaming/km/rest/api/v3/enterprise/rebalance/ClusterBalanceController.java create mode 100644 km-task/src/main/java/com/xiaojukeji/know/streaming/km/task/enterprise/rebalance/job/ClusterBalanceJobTask.java diff --git a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/cluster/impl/MultiClusterPhyManagerImpl.java b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/cluster/impl/MultiClusterPhyManagerImpl.java index 9d1f7e33..68dd4ac7 100644 --- a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/cluster/impl/MultiClusterPhyManagerImpl.java +++ b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/cluster/impl/MultiClusterPhyManagerImpl.java @@ -176,7 +176,10 @@ public class MultiClusterPhyManagerImpl implements MultiClusterPhyManager { // 获取所有的metrics List metricsList = new ArrayList<>(); for (ClusterPhyDashboardVO vo: voList) { - metricsList.add(clusterMetricService.getLatestMetricsFromCache(vo.getId())); + ClusterMetrics clusterMetrics = clusterMetricService.getLatestMetricsFromCache(vo.getId()); + clusterMetrics.getMetrics().putIfAbsent(ClusterMetricVersionItems.CLUSTER_METRIC_HEALTH_STATE, (float) HealthStateEnum.UNKNOWN.getDimension()); + + metricsList.add(clusterMetrics); } // 范围搜索 diff --git a/km-common/pom.xml b/km-common/pom.xml index e9e66c22..2e1edb75 100644 --- a/km-common/pom.xml +++ b/km-common/pom.xml @@ -22,6 +22,12 @@ + + com.xiaojukeji.kafka + km-rebalance + ${project.parent.version} + + org.springframework spring-web diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceIntervalDTO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceIntervalDTO.java new file mode 100644 index 00000000..0f198056 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceIntervalDTO.java @@ -0,0 +1,26 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + + +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceIntervalDTO { + @NotBlank(message = "clusterBalanceIntervalDTO.type不允许为空") + @ApiModelProperty("均衡维度:cpu,disk,bytesIn,bytesOut") + private String type; + + @NotNull(message = "clusterBalanceIntervalDTO.intervalPercent不允许为空") + @ApiModelProperty("平衡区间百分比") + private Double intervalPercent; + + @NotNull(message = "clusterBalanceIntervalDTO.priority不允许为空") + @ApiModelProperty("优先级") + private Integer priority; + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceOverviewDTO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceOverviewDTO.java new file mode 100644 index 00000000..7aa8a5f0 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceOverviewDTO.java @@ -0,0 +1,20 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Map; + + +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceOverviewDTO extends PaginationBaseDTO { + + @ApiModelProperty("host") + private String host; + + @ApiModelProperty("key:disk,bytesOut,bytesIn value:均衡状态 0：已均衡；2：未均衡") + private Map stateParam; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalancePreviewDTO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalancePreviewDTO.java new file mode 100644 index 00000000..705178ac --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalancePreviewDTO.java @@ -0,0 +1,43 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + + +/** + * @author zengqiao + * @date 22/02/24 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalancePreviewDTO extends BaseDTO { + + @ApiModelProperty("集群id") + private Long clusterId; + + @ApiModelProperty("均衡节点") + private List brokers; + + @ApiModelProperty("topic黑名单") + private List topicBlackList; + + @ApiModelProperty("均衡区间详情") + private List clusterBalanceIntervalList; + + @ApiModelProperty("指标计算周期，单位分钟") + private Integer metricCalculationPeriod; + + @ApiModelProperty("任务并行数") + private Integer parallelNum; + + @ApiModelProperty("执行策略， 1：优先最大副本，2：优先最小副本") + private Integer executionStrategy; + + @ApiModelProperty("限流值") + private Long throttleUnitB; + + } diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceStrategyDTO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceStrategyDTO.java new file mode 100644 index 00000000..6eff2c5d --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/dto/ClusterBalanceStrategyDTO.java @@ -0,0 +1,66 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.util.List; + + +/** + * @author zengqiao + * @date 22/02/24 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceStrategyDTO extends BaseDTO { + + @ApiModelProperty("是否是周期性任务") + private boolean scheduleJob; + + @NotBlank(message = "scheduleCron不允许为空") + @ApiModelProperty("如果是周期任务，那么任务的周期cron表达式") + private String scheduleCron; + + @NotNull(message = "status不允许为空") + @ApiModelProperty("周期任务状态：0:不开启，1：开启") + private Integer status; + + @NotNull(message = "clusterId不允许为空") + @ApiModelProperty("集群id") + private Long clusterId; + + @ApiModelProperty("均衡节点") + private List brokers; + + @ApiModelProperty("topic黑名单") + private List topicBlackList; + + @NotNull(message = "clusterBalanceIntervalDTO不允许为空") + @ApiModelProperty("均衡区间详情") + private List clusterBalanceIntervalList; + + @NotNull(message = "metricCalculationPeriod不允许为空") + @ApiModelProperty("指标计算周期，单位秒") + private Integer metricCalculationPeriod; + + @NotNull(message = "parallelNum不允许为空") + @ApiModelProperty("任务并行数（0代表不限）") + private Integer parallelNum; + + @NotNull(message = "executionStrategy不允许为空") + @ApiModelProperty("执行策略， 1：优先最大副本，2：优先最小副本") + private Integer executionStrategy; + + @Min(value = 1, message = "throttleUnitB不允许小于1") + @ApiModelProperty("限流值") + private Long throttleUnitB; + + @ApiModelProperty("备注说明") + private String description; + + } diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceInterval.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceInterval.java new file mode 100644 index 00000000..3a05160b --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceInterval.java @@ -0,0 +1,27 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@EnterpriseLoadReBalance +public class ClusterBalanceInterval { + /** + * 均衡维度:cpu,disk,bytesIn,bytesOut + */ + private String type; + + /** + * 平衡区间百分比 + */ + private Double intervalPercent; + + /** + * 优先级 + */ + private Integer priority; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceItemState.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceItemState.java new file mode 100644 index 00000000..04933dce --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/ClusterBalanceItemState.java @@ -0,0 +1,41 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@EnterpriseLoadReBalance +public class ClusterBalanceItemState { + + /** + * 是否配置集群平衡:true:已配置，false:未配置 + */ + private Boolean configureBalance; + + /** + * 是否开启均衡:true:开启，false: 未开启 + */ + private Boolean enable; + + /** + * 子项是否均衡:key: disk,bytesIn,bytesOut,cpu ; value:true:已均衡，false:未均衡 + * @see com.xiaojukeji.know.streaming.km.rebalance.model.Resource + */ + private Map itemState; + + public Integer getResItemState(Resource res) { + if (itemState == null || !itemState.containsKey(res.resource())) { + return Constant.INVALID_CODE; + } + + return itemState.get(res.resource()) ? Constant.YES: Constant.NO; + } +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceJobConfig.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceJobConfig.java new file mode 100644 index 00000000..9b2d5ca7 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceJobConfig.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.entity.BaseEntity; +import lombok.Data; + +/** + * 集群均衡任务 实体类 + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceJobConfig extends BaseEntity { + + /** + * 序列化版本号 + */ + private static final long serialVersionUID=1L; + + /** + * 集群id + */ + private Long clusterId; + + /** + * 均衡节点 + */ + private String brokers; + + /** + * topic黑名单 + */ + private String topicBlackList; + + /** + * 1:立即均衡，2：周期均衡 + */ + private Integer type; + + /** + * 任务周期 + */ + private String taskCron; + + /** + * 均衡区间详情 + */ + private String balanceIntervalJson; + + /** + * 指标计算周期，单位分钟 + */ + private Integer metricCalculationPeriod; + + /** + * 迁移脚本 + */ + private String reassignmentJson; + + /** + * 任务并行数 + */ + private Integer parallelNum; + + /** + * 执行策略， 1：优先最大副本，2：优先最小副本 + */ + private Integer executionStrategy; + + /** + * 限流值 + */ + private Long throttleUnitByte; + + /** + * 操作人 + */ + private String creator; + + /** + * 任务状态 0：未开启，1：开启 + */ + private Integer status; + +} + diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassign.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassign.java new file mode 100644 index 00000000..d3a0a8e3 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassign.java @@ -0,0 +1,64 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Data; + +import java.util.Date; + +/** + * @author zengqiao + * @date 22/05/06 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceReassign { + /** + * jobID + */ + private Long jobId; + + /** + * 集群id + */ + private Long clusterId; + + /** + * Topic名称 + */ + private String topicName; + + /** + * 分区ID + */ + private Integer partitionId; + + /** + * 源BrokerId列表 + */ + private String originalBrokerIds; + + /** + * 目标BrokerId列表 + */ + private String reassignBrokerIds; + + /** + * 任务开始时间 + */ + private Date startTime; + + /** + * 任务完成时间 + */ + private Date finishedTime; + + /** + * 扩展数据 + */ + private String extendData; + + /** + * 任务状态 + */ + private Integer status; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignDetail.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignDetail.java new file mode 100644 index 00000000..f64ab6ce --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignDetail.java @@ -0,0 +1,36 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail.ClusterBalanceDetailDataGroupByTopic; +import lombok.Data; + +import java.util.Date; +import java.util.List; + +/** + * @author zengqiao + * @date 22/05/06 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceReassignDetail { + /** + * 限流值 + */ + private Long throttleUnitB; + + /** + * 开始时间 + */ + private Date startTime; + + /** + * 完成时间 + */ + private Date finishedTime; + + /** + * 详细信息 + */ + private List reassignTopicDetailsList; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignExtendData.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignExtendData.java new file mode 100644 index 00000000..4017c7ac --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/ClusterBalanceReassignExtendData.java @@ -0,0 +1,47 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Data; + +/** + * @author zengqiao + * @date 22/05/06 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceReassignExtendData { + /** + * 原本保存时间 + */ + private Long originalRetentionTimeUnitMs; + + /** + * 迁移时保存时间 + */ + private Long reassignRetentionTimeUnitMs; + + /** + * 需迁移LogSize + */ + private Long needReassignLogSizeUnitB; + + /** + * 已完成迁移LogSize + */ + private Long finishedReassignLogSizeUnitB; + + /** + * 预计剩余时长 + */ + private Long remainTimeUnitMs; + + /** + * 当前副本数 + */ + private Integer originReplicaNum; + + /** + * 新的副本数 + */ + private Integer reassignReplicaNum; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/content/JobClusterBalanceContent.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/content/JobClusterBalanceContent.java new file mode 100644 index 00000000..8ed77bfb --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/content/JobClusterBalanceContent.java @@ -0,0 +1,43 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.content; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.content.BaseJobCreateContent; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceIntervalDTO; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.Min; +import java.util.List; + +@Data +@EnterpriseLoadReBalance +public class JobClusterBalanceContent extends BaseJobCreateContent { + @Min(value = 1, message = "clusterId不允许为null或者小于0") + @ApiModelProperty(value = "集群ID, 默认为逻辑集群ID", example = "6") + private Long clusterId; + + @Min(value = 1, message = "throttle不允许为null或者小于0") + @ApiModelProperty(value = "限流值", example = "102400000") + private Long throttleUnitB; + + @ApiModelProperty("topic黑名单") + private List topicBlackList; + + @ApiModelProperty("均衡区间详情") + private List clusterBalanceIntervalList; + + @ApiModelProperty("指标计算周期，单位分钟") + private Integer metricCalculationPeriod; + + @ApiModelProperty("任务并行数") + private Integer parallelNum; + + @ApiModelProperty("执行策略， 1：优先最大副本，2：优先最小副本") + private Integer executionStrategy; + + @ApiModelProperty("备注说明") + private String description; + + @ApiModelProperty("是否是周期性任务") + private boolean scheduleJob; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/AbstractClusterBalanceDetailData.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/AbstractClusterBalanceDetailData.java new file mode 100644 index 00000000..f4b7daa3 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/AbstractClusterBalanceDetailData.java @@ -0,0 +1,79 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Data; + +import java.util.List; + +/** + * @author zengqiao + * @date 22/05/06 + */ +@Data +@EnterpriseLoadReBalance +public abstract class AbstractClusterBalanceDetailData { + /** + * 物流集群ID + */ + private Long clusterPhyId; + + /** + * Topic名称 + */ + private String topicName; + + /** + * 源Broker列表 + */ + private List originalBrokerIdList; + + /** + * 目标Broker列表 + */ + private List reassignBrokerIdList; + + /** + * 需迁移LogSize + */ + private Long needReassignLogSizeUnitB; + + /** + * 已完成迁移LogSize + */ + private Long finishedReassignLogSizeUnitB; + + /** + * 预计剩余时长 + */ + private Long remainTimeUnitMs; + + /** + * 当前副本数 + */ + private Integer presentReplicaNum; + + /** + * 新的副本数 + */ + private Integer oldReplicaNum; + + /** + * 新的副本数 + */ + private Integer newReplicaNum; + + /** + * 原本保存时间 + */ + private Long originalRetentionTimeUnitMs; + + /** + * 迁移时保存时间 + */ + private Long reassignRetentionTimeUnitMs; + + /** + * 状态 + */ + private Integer status; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByPartition.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByPartition.java new file mode 100644 index 00000000..45837a75 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByPartition.java @@ -0,0 +1,17 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Data; + +/** + * @author zengqiao + * @date 22/05/06 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceDetailDataGroupByPartition extends AbstractClusterBalanceDetailData { + /** + * 分区ID + */ + private Integer partitionId; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByTopic.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByTopic.java new file mode 100644 index 00000000..8c72827f --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalanceDetailDataGroupByTopic.java @@ -0,0 +1,22 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Data; + +import java.util.List; + + +/** + * @author zengqiao + * @date 22/05/06 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceDetailDataGroupByTopic extends AbstractClusterBalanceDetailData { + /** + * 分区ID列表 + */ + private List partitionIdList; + + private List reassignPartitionDetailsList; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalancePlanDetail.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalancePlanDetail.java new file mode 100644 index 00000000..e2528e35 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/entity/job/detail/ClusterBalancePlanDetail.java @@ -0,0 +1,76 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡详细信息") +public class ClusterBalancePlanDetail implements Serializable { + @ApiModelProperty(value = "是否均衡，1：已均衡；2：未均衡") + private Integer status; + + @ApiModelProperty(value = "brokerId") + private Integer brokerId; + + @ApiModelProperty(value = "broker host") + private String host; + + @ApiModelProperty(value = "均衡前 cpu") + private Double cpuBefore; + + @ApiModelProperty(value = "均衡前 disk") + private Double diskBefore; + + @ApiModelProperty(value = "均衡前 byteIn") + private Double byteInBefore; + + @ApiModelProperty(value = "均衡前 byteOut") + private Double byteOutBefore; + + @ApiModelProperty(value = "均衡后 cpu") + private Double cpuAfter; + + @ApiModelProperty(value = "是否均衡，1：已均衡；2：未均衡") + private Integer cpuStatus; + + @ApiModelProperty(value = "均衡后 disk") + private Double diskAfter; + + @ApiModelProperty(value = "是否均衡，1：已均衡；2：未均衡") + private Integer diskStatus; + + @ApiModelProperty(value = "均衡后 byteIn") + private Double byteInAfter; + + @ApiModelProperty(value = "是否均衡，1：已均衡；2：未均衡") + private Integer byteInStatus; + + @ApiModelProperty(value = "均衡后 byteOut") + private Double byteOutAfter; + + @ApiModelProperty(value = "是否均衡，1：已均衡；2：未均衡") + private Integer byteOutStatus; + + @ApiModelProperty(value = "均衡流入大小") + private Double inSize; + + @ApiModelProperty(value = "均衡流入副本个数") + private Double inReplica; + + @ApiModelProperty(value = "均衡流出大小") + private Double outSize; + + @ApiModelProperty(value = "均衡流出副本个数") + private Double outReplica; + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobConfigPO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobConfigPO.java new file mode 100644 index 00000000..6462a500 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobConfigPO.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po; + +import com.baomidou.mybatisplus.annotation.TableName; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 集群均衡任务 实体类 + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Data +@EnterpriseLoadReBalance +@NoArgsConstructor +@TableName(Constant.MYSQL_TABLE_NAME_PREFIX + "cluster_balance_job_config") +public class ClusterBalanceJobConfigPO extends BasePO { + +/** + * 序列化版本号 + */ +private static final long serialVersionUID=1L; + + /** + * 集群id + */ + private Long clusterId; + + /** + * topic黑名单 + */ + private String topicBlackList; + + /** + * 任务周期 + */ + private String taskCron; + + /** + * 均衡区间详情 + */ + private String balanceIntervalJson; + + /** + * 指标计算周期，单位分钟 + */ + private Integer metricCalculationPeriod; + + /** + * 迁移脚本 + */ + private String reassignmentJson; + + /** + * 任务并行数 + */ + private Integer parallelNum; + + /** + * 执行策略， 1：优先最大副本，2：优先最小副本 + */ + private Integer executionStrategy; + + /** + * 限流值 + */ + private Long throttleUnitB; + + /** + * 操作人 + */ + private String creator; + + /** + * 任务状态 0：未开启，1：开启 + */ + private Integer status; +} + diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobPO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobPO.java new file mode 100644 index 00000000..10866857 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceJobPO.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po; + +import com.baomidou.mybatisplus.annotation.TableName; +import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +/** + * 集群均衡任务 实体类 + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Data +@NoArgsConstructor +@TableName(Constant.MYSQL_TABLE_NAME_PREFIX + "cluster_balance_job") +public class ClusterBalanceJobPO extends BasePO { + + /** + * 序列化版本号 + */ + private static final long serialVersionUID=1L; + + /** + * 集群id + */ + private Long clusterId; + + /** + * 均衡节点 + */ + private String brokers; + + /** + * topic黑名单 + */ + private String topicBlackList; + + /** + * 1:立即均衡，2：周期均衡 + */ + private Integer type; + + /** + * 均衡区间详情 + */ + private String balanceIntervalJson; + + /** + * 指标计算周期，单位分钟 + */ + private Integer metricCalculationPeriod; + + /** + * 迁移脚本 + */ + private String reassignmentJson; + + /** + * 任务并行数 + */ + private Integer parallelNum; + + /** + * 执行策略， 1：优先最大副本，2：优先最小副本 + */ + private Integer executionStrategy; + + /** + * 限流值 + */ + private Long throttleUnitB; + + /** + * 总迁移大小 + */ + private Double totalReassignSize; + + /** + * 总迁移副本数 + */ + private Integer totalReassignReplicaNum; + + /** + * 移入topic + */ + private String moveInTopicList; + + /** + * 节点均衡详情 + */ + private String brokerBalanceDetail; + + /** + * 任务状态 1：进行中，2：准备，3，成功，4：失败，5：取消 + */ + private Integer status; + + /** + * 操作人 + */ + private String creator; + + /** + * 任务开始时间 + */ + private Date startTime; + + /** + * 任务完成时间 + */ + private Date finishedTime; + + /** + * 备注说明 + */ + private String description; +} + diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceReassignPO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceReassignPO.java new file mode 100644 index 00000000..7cc4ef65 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/po/ClusterBalanceReassignPO.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po; + +import com.baomidou.mybatisplus.annotation.TableName; +import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +/** + * 集群平衡迁移详情 实体类 + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Data +@NoArgsConstructor +@TableName(Constant.MYSQL_TABLE_NAME_PREFIX + "cluster_balance_reassign") +public class ClusterBalanceReassignPO extends BasePO { + + /** + * 序列化版本号 + */ + private static final long serialVersionUID=1L; + /** + * jobID + */ + private Long jobId; + + /** + * 集群id + */ + private Long clusterId; + + /** + * Topic名称 + */ + private String topicName; + + /** + * 分区ID + */ + private Integer partitionId; + + /** + * 源BrokerId列表 + */ + private String originalBrokerIds; + + /** + * 目标BrokerId列表 + */ + private String reassignBrokerIds; + + /** + * 任务开始时间 + */ + private Date startTime; + + /** + * 任务完成时间 + */ + private Date finishedTime; + + /** + * 扩展数据 + */ + private String extendData; + + /** + * 任务状态 + */ + private Integer status; + +} + diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistorySubVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistorySubVO.java new file mode 100644 index 00000000..80bbc668 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistorySubVO.java @@ -0,0 +1,24 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡历史信息") +public class ClusterBalanceHistorySubVO implements Serializable { + @ApiModelProperty(value = "均衡成功节点数") + private Long successNu; + + @ApiModelProperty(value = "未均衡成功节点数") + private Long failedNu; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistoryVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistoryVO.java new file mode 100644 index 00000000..fb71e1eb --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceHistoryVO.java @@ -0,0 +1,34 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.Date; +import java.util.Map; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡历史信息") +public class ClusterBalanceHistoryVO implements Serializable { + @ApiModelProperty(value = "均衡开始执行时间") + private Date begin; + + @ApiModelProperty(value = "均衡执行结束时间") + private Date end; + + @ApiModelProperty(value = "均衡任务id") + private Long jobId; + + @ApiModelProperty(value = "子项均衡历史信息", example = "cpu、disk") + private Map sub; + + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceIntervalVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceIntervalVO.java new file mode 100644 index 00000000..d3d28957 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceIntervalVO.java @@ -0,0 +1,20 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + + +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceIntervalVO { + @ApiModelProperty("均衡维度:cpu,disk,bytesIn,bytesOut") + private String type; + + @ApiModelProperty("平衡区间百分比") + private Double intervalPercent; + + @ApiModelProperty("优先级") + private Integer priority; + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceJobConfigVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceJobConfigVO.java new file mode 100644 index 00000000..08c36124 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceJobConfigVO.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.ClusterBalanceInterval; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * 集群均衡任务 实体类 + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Data +@EnterpriseLoadReBalance +public class ClusterBalanceJobConfigVO { + /** + * 序列化版本号 + */ + private static final long serialVersionUID=1L; + + @ApiModelProperty("集群id") + private Long clusterId; + + @ApiModelProperty("topic黑名单") + private List topicBlackList; + + @ApiModelProperty("任务周期") + private String scheduleCron; + + @ApiModelProperty("均衡区间详情") + private List clusterBalanceIntervalList; + + @ApiModelProperty("指标计算周期，单位分钟") + private Integer metricCalculationPeriod; + + @ApiModelProperty("任务并行数") + private Integer parallelNum; + + @ApiModelProperty("执行策略， 1：优先最大副本，2：优先最小副本") + private Integer executionStrategy; + + @ApiModelProperty("限流值") + private Long throttleUnitB; + + @ApiModelProperty("任务状态 0：未开启，1：开启") + private Integer status; + +} + diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewSubVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewSubVO.java new file mode 100644 index 00000000..770e274f --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewSubVO.java @@ -0,0 +1,32 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡列表信息") +public class ClusterBalanceOverviewSubVO implements Serializable { + @ApiModelProperty(value = "平均值", example = "cpu的平均值，43.4") + private Double avg; + + @ApiModelProperty(value = "规格", example = "1000") + private Double spec; + + @ApiModelProperty(value = "均衡状态", example = "0:已均衡，-1:低于均衡值，1高于均衡值") + private Integer status ; + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewVO.java new file mode 100644 index 00000000..872b6c0a --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceOverviewVO.java @@ -0,0 +1,37 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡列表信息") +public class ClusterBalanceOverviewVO implements Serializable { + @ApiModelProperty(value = "brokerId", example = "123") + private Integer brokerId; + + @ApiModelProperty(value = "broker host") + private String host; + + @ApiModelProperty(value = "broker 对应的 rack") + private String rack; + + @ApiModelProperty(value = "leader") + private Integer leader; + + @ApiModelProperty(value = "replicas") + private Integer replicas; + + @ApiModelProperty(value = "子项统计详细信息", example = "cpu、disk") + private Map sub; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanDetailVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanDetailVO.java new file mode 100644 index 00000000..eb41ae7f --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanDetailVO.java @@ -0,0 +1,64 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡历史信息") +public class ClusterBalancePlanDetailVO implements Serializable { + @ApiModelProperty(value = "是否均衡，0：已均衡；2：未均衡") + private Integer status; + + @ApiModelProperty(value = "brokerId") + private Integer brokerId; + + @ApiModelProperty(value = "broker host") + private String host; + + @ApiModelProperty(value = "均衡前 cpu") + private Double cpuBefore; + + @ApiModelProperty(value = "均衡前 disk") + private Double diskBefore; + + @ApiModelProperty(value = "均衡前 byteIn") + private Double byteInBefore; + + @ApiModelProperty(value = "均衡前 byteOut") + private Double byteOutBefore; + + @ApiModelProperty(value = "均衡后 cpu") + private Double cpuAfter; + + @ApiModelProperty(value = "均衡后 disk") + private Double diskAfter; + + @ApiModelProperty(value = "均衡后 byteIn") + private Double byteInAfter; + + @ApiModelProperty(value = "均衡后 byteOut") + private Double byteOutAfter; + + @ApiModelProperty(value = "均衡流入大小") + private Double inSize; + + @ApiModelProperty(value = "均衡流入副本个数") + private Double inReplica; + + @ApiModelProperty(value = "均衡流出大小") + private Double outSize; + + @ApiModelProperty(value = "均衡流出副本个数") + private Double outReplica; + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanVO.java new file mode 100644 index 00000000..dd624d79 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalancePlanVO.java @@ -0,0 +1,49 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡信息") +public class ClusterBalancePlanVO implements Serializable { + @ApiModelProperty(value = "均衡计划类型，1：立即均衡；2：周期均衡") + private Integer type; + + @ApiModelProperty(value = "均衡执行的节点范围") + private List brokers; + + @ApiModelProperty(value = "均衡执行的Topic黑名单") + private List blackTopics; + + @ApiModelProperty(value = "均衡执行移入的Topic名单") + private List topics; + + @ApiModelProperty(value = "均衡总迁移的磁盘大小，单位byte") + private Double moveSize; + + @ApiModelProperty(value = "均衡总迁移的副本个数") + private Integer replicas; + + @ApiModelProperty(value = "均衡阈值") + private String threshold; + + @ApiModelProperty(value = "reassignment json") + private String reassignmentJson; + + @ApiModelProperty(value = "均衡区间信息") + private List clusterBalanceIntervalList; + + @ApiModelProperty(value = "均衡计划明细") + private List detail; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateSubVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateSubVO.java new file mode 100644 index 00000000..e713ca20 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateSubVO.java @@ -0,0 +1,32 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@EnterpriseLoadReBalance +@AllArgsConstructor +@NoArgsConstructor +@ApiModel(description = "集群均衡状态子项的详细统计信息") +public class ClusterBalanceStateSubVO { + + @ApiModelProperty(value = "平均值", example = "cpu的平均值，43.4") + private Double avg; + + @ApiModelProperty(value = "周期均衡时的均衡区间", example = "cpu的均衡值") + private Double interval; + + @ApiModelProperty(value = "处于周期均衡时的均衡区间的最小值以下的broker个数", example = "4") + private Long smallNu; + + @ApiModelProperty(value = "处于周期均衡时的均衡区间的broker个数", example = "4") + private Long betweenNu; + + @ApiModelProperty(value = "处于周期均衡时的均衡区间的最大值以上的broker个数", example = "4") + private Long bigNu; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateVO.java new file mode 100644 index 00000000..3803315b --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/bean/vo/ClusterBalanceStateVO.java @@ -0,0 +1,32 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.Date; +import java.util.Map; + +/** + * 集群Topic信息 + * @author zengqiao + * @date 22/02/23 + */ +@Data +@EnterpriseLoadReBalance +@ApiModel(description = "集群均衡状态信息") +public class ClusterBalanceStateVO implements Serializable { + @ApiModelProperty(value = "均衡状态", example = "0:已均衡，2:未均衡") + private Integer status; + + @ApiModelProperty(value = "是否开启均衡", example = "true:开启，false:未开启") + private Boolean enable; + + @ApiModelProperty(value = "下次均衡开始时间") + private Date next; + + @ApiModelProperty(value = "子项统计详细信息", example = "cpu、disk") + private Map sub; +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceConverter.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceConverter.java new file mode 100644 index 00000000..f65f9035 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceConverter.java @@ -0,0 +1,476 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceIntervalDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalancePreviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceStrategyDTO; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec; +import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.ClusterBalanceInterval; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.ClusterBalanceReassignExtendData; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail.ClusterBalancePlanDetail; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.content.JobClusterBalanceContent; +import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobConfigPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.*; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.enums.ClusterBalanceStateEnum; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.enums.ClusterBalanceTypeEnum; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum; +import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.*; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import org.apache.kafka.clients.CommonClientConfigs; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.PARTITION_INDEX; + +@EnterpriseLoadReBalance +public class ClusterBalanceConverter { + + private ClusterBalanceConverter() { + } + + public static BalanceParameter convert2BalanceParameter(ClusterBalanceJobConfigPO configPO, Map brokerMap, Map brokerSpecMap, ClusterPhy clusterPhy, String esUrl, List topicNames) { + BalanceParameter balanceParameter = new BalanceParameter(); + List clusterBalanceIntervalDTOS = ConvertUtil.str2ObjArrayByJson(configPO.getBalanceIntervalJson(), ClusterBalanceIntervalDTO.class); + + List goals = new ArrayList<>(); + for(ClusterBalanceIntervalDTO clusterBalanceIntervalDTO : clusterBalanceIntervalDTOS){ + if (Resource.DISK.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setDiskThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.DISK.goal()); + }else if (Resource.CPU.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setCpuThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + // todo cpu底层暂未实现，先不加goal + }else if (Resource.NW_IN.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkInThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_IN.goal()); + }else if (Resource.NW_OUT.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkOutThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_OUT.goal()); + } + } + balanceParameter.setGoals(goals); + balanceParameter.setCluster(clusterPhy.getId().toString()); + balanceParameter.setExcludedTopics(configPO.getTopicBlackList()); + balanceParameter.setEsIndexPrefix(PARTITION_INDEX + "_"); + balanceParameter.setEsRestURL(esUrl); + balanceParameter.setBalanceBrokers(CommonUtils.intSet2String(brokerMap.keySet())); + balanceParameter.setHardwareEnv(convert2ListHostEnv(brokerMap, brokerSpecMap)); + balanceParameter.setBeforeSeconds(configPO.getMetricCalculationPeriod()); + balanceParameter.setIgnoredTopics(CommonUtils.strList2String(topicNames)); + + Properties kafkaConfig = new Properties(); + kafkaConfig.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, clusterPhy.getBootstrapServers()); + kafkaConfig.putAll(ConvertUtil.str2ObjByJson(clusterPhy.getClientProperties(), Properties.class)); + balanceParameter.setKafkaConfig(kafkaConfig); + return balanceParameter; + + } + + public static BalanceParameter convert2BalanceParameter(ClusterBalanceJobPO clusterBalanceJobPO, Map brokerMap, Map brokerSpecMap, ClusterPhy clusterPhy, String esUrl, List topicNames) { + BalanceParameter balanceParameter = new BalanceParameter(); + List clusterBalanceIntervalDTOS = ConvertUtil.str2ObjArrayByJson(clusterBalanceJobPO.getBalanceIntervalJson(), ClusterBalanceIntervalDTO.class); + + List goals = new ArrayList<>(); + for(ClusterBalanceIntervalDTO clusterBalanceIntervalDTO : clusterBalanceIntervalDTOS){ + if (Resource.DISK.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setDiskThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.DISK.goal()); + }else if (Resource.CPU.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setCpuThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + // todo cpu底层暂未实现，先不加goal + }else if (Resource.NW_IN.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkInThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_IN.goal()); + }else if (Resource.NW_OUT.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkOutThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_OUT.goal()); + } + } + balanceParameter.setGoals(goals); + balanceParameter.setCluster(clusterPhy.getId().toString()); + balanceParameter.setExcludedTopics(clusterBalanceJobPO.getTopicBlackList()); + balanceParameter.setEsIndexPrefix(PARTITION_INDEX + "_"); + balanceParameter.setEsRestURL(esUrl); + balanceParameter.setBalanceBrokers(clusterBalanceJobPO.getBrokers()); + balanceParameter.setHardwareEnv(convert2ListHostEnv(brokerMap, brokerSpecMap)); + balanceParameter.setBeforeSeconds(clusterBalanceJobPO.getMetricCalculationPeriod()); + balanceParameter.setIgnoredTopics(CommonUtils.strList2String(topicNames)); + + Properties kafkaConfig = new Properties(); + kafkaConfig.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, clusterPhy.getBootstrapServers()); + kafkaConfig.putAll(ConvertUtil.str2ObjByJson(clusterPhy.getClientProperties(), Properties.class)); + balanceParameter.setKafkaConfig(kafkaConfig); + return balanceParameter; + + } + + public static BalanceParameter convert2BalanceParameter(JobClusterBalanceContent dto, List brokers, Map brokerSpecMap, ClusterPhy clusterPhy, String esUrl, List topicNames) { + BalanceParameter balanceParameter = new BalanceParameter(); + List clusterBalanceIntervalDTOS = dto.getClusterBalanceIntervalList().stream() + .sorted(Comparator.comparing(ClusterBalanceIntervalDTO::getPriority)).collect(Collectors.toList()); + List goals = new ArrayList<>(); + for(ClusterBalanceIntervalDTO clusterBalanceIntervalDTO : clusterBalanceIntervalDTOS){ + if (Resource.DISK.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setDiskThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.DISK.goal()); + }else if (Resource.CPU.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setCpuThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + // todo cpu底层暂未实现，先不加goal + }else if (Resource.NW_IN.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkInThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_IN.goal()); + }else if (Resource.NW_OUT.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkOutThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_OUT.goal()); + } + } + + Map brokerMap = brokers.stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity())); + balanceParameter.setGoals(goals); + balanceParameter.setCluster(clusterPhy.getId().toString()); + balanceParameter.setExcludedTopics(CommonUtils.strList2String(dto.getTopicBlackList())); + balanceParameter.setEsIndexPrefix(PARTITION_INDEX + "_"); + balanceParameter.setEsRestURL(esUrl); + balanceParameter.setBalanceBrokers(CommonUtils.intSet2String(brokerMap.keySet())); + balanceParameter.setHardwareEnv(convert2ListHostEnv(brokerMap, brokerSpecMap)); + balanceParameter.setBeforeSeconds(dto.getMetricCalculationPeriod()); + balanceParameter.setIgnoredTopics(CommonUtils.strList2String(topicNames)); + + Properties kafkaConfig = new Properties(); + kafkaConfig.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, clusterPhy.getBootstrapServers()); + kafkaConfig.putAll(ConvertUtil.str2ObjByJson(clusterPhy.getClientProperties(), Properties.class)); + balanceParameter.setKafkaConfig(kafkaConfig); + return balanceParameter; + + } + + public static BalanceParameter convert2BalanceParameter(ClusterBalancePreviewDTO dto, Map brokerMap, Map brokerSpecMap, ClusterPhy clusterPhy, String esUrl, List topicNames) { + BalanceParameter balanceParameter = new BalanceParameter(); + List clusterBalanceIntervalDTOS = dto.getClusterBalanceIntervalList().stream() + .sorted(Comparator.comparing(ClusterBalanceIntervalDTO::getPriority)).collect(Collectors.toList()); + List goals = new ArrayList<>(); + for(ClusterBalanceIntervalDTO clusterBalanceIntervalDTO : clusterBalanceIntervalDTOS){ + if (Resource.DISK.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setDiskThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.DISK.goal()); + }else if (Resource.CPU.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setCpuThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + // todo cpu底层暂未实现，先不加goal + }else if (Resource.NW_IN.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkInThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_IN.goal()); + }else if (Resource.NW_OUT.resource().equals(clusterBalanceIntervalDTO.getType())){ + balanceParameter.setNetworkOutThreshold(clusterBalanceIntervalDTO.getIntervalPercent()/100); + goals.add(BalanceGoal.NW_OUT.goal()); + } + } + balanceParameter.setGoals(goals); + balanceParameter.setCluster(clusterPhy.getId().toString()); + balanceParameter.setExcludedTopics(CommonUtils.strList2String(dto.getTopicBlackList())); + balanceParameter.setEsIndexPrefix(PARTITION_INDEX + "_"); + balanceParameter.setEsRestURL(esUrl); + balanceParameter.setBalanceBrokers(CommonUtils.intList2String(dto.getBrokers())); + balanceParameter.setHardwareEnv(convert2ListHostEnv(brokerMap, brokerSpecMap)); + balanceParameter.setBeforeSeconds(dto.getMetricCalculationPeriod()); + balanceParameter.setIgnoredTopics(CommonUtils.strList2String(topicNames)); + + Properties kafkaConfig = new Properties(); + kafkaConfig.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, clusterPhy.getBootstrapServers()); + kafkaConfig.putAll(ConvertUtil.str2ObjByJson(clusterPhy.getClientProperties(), Properties.class)); + balanceParameter.setKafkaConfig(kafkaConfig); + return balanceParameter; + + } + + + public static ClusterBalanceJobPO convert2ClusterBalanceJobPO(Long jobId, JobClusterBalanceContent jobDTO, OptimizerResult optimizerResult, List brokers, String operator, String json) { + if (ValidateUtils.anyNull(jobDTO, optimizerResult, optimizerResult.resultJsonOverview(), + optimizerResult.resultJsonDetailed(), optimizerResult.resultDetailed(), optimizerResult.resultJsonTask())){ + return null; + } + + ClusterBalanceJobPO clusterBalanceJobPO = new ClusterBalanceJobPO(); + clusterBalanceJobPO.setId(jobId); + clusterBalanceJobPO.setType(jobDTO.isScheduleJob()? + ClusterBalanceTypeEnum.CYCLE.getType():ClusterBalanceTypeEnum.IMMEDIATELY.getType()); + clusterBalanceJobPO.setStatus(JobStatusEnum.WAITING.getStatus()); + clusterBalanceJobPO.setCreator(operator); + clusterBalanceJobPO.setParallelNum(jobDTO.getParallelNum()); + clusterBalanceJobPO.setThrottleUnitB(jobDTO.getThrottleUnitB()); + clusterBalanceJobPO.setDescription(jobDTO.getDescription()); + clusterBalanceJobPO.setBrokers(CommonUtils.intList2String(brokers.stream().map(Broker::getBrokerId).collect(Collectors.toList()))); + clusterBalanceJobPO.setClusterId(jobDTO.getClusterId()); + clusterBalanceJobPO.setTopicBlackList(CommonUtils.strList2String(jobDTO.getTopicBlackList())); + clusterBalanceJobPO.setMoveInTopicList(optimizerResult.resultOverview().getMoveTopics()); + clusterBalanceJobPO.setExecutionStrategy(jobDTO.getExecutionStrategy()); + clusterBalanceJobPO.setBalanceIntervalJson(ConvertUtil.obj2Json(jobDTO.getClusterBalanceIntervalList())); + clusterBalanceJobPO.setBrokerBalanceDetail(ConvertUtil.obj2Json(convert2ClusterBalancePlanDetail(optimizerResult.resultDetailed()))); + clusterBalanceJobPO.setMetricCalculationPeriod(jobDTO.getMetricCalculationPeriod()); + clusterBalanceJobPO.setReassignmentJson(json); + clusterBalanceJobPO.setTotalReassignSize(optimizerResult.resultOverview().getTotalMoveSize()); + clusterBalanceJobPO.setTotalReassignReplicaNum(optimizerResult.resultOverview().getMoveReplicas()); + clusterBalanceJobPO.setDescription(optimizerResult.resultJsonBalanceActionHistory()); + return clusterBalanceJobPO; + } + + public static ClusterBalanceReassignPO convert2ClusterBalanceReassignPO(BalanceTask balanceTask, Topic topic, Long jobId, Long clusterId) { + ClusterBalanceReassignPO reassignPO = new ClusterBalanceReassignPO(); + reassignPO.setClusterId(clusterId); + reassignPO.setJobId(jobId); + reassignPO.setPartitionId(balanceTask.getPartition()); + reassignPO.setOriginalBrokerIds(CommonUtils.intList2String(topic.getPartitionMap().get(balanceTask.getPartition()))); + reassignPO.setReassignBrokerIds(CommonUtils.intList2String(balanceTask.getReplicas())); + reassignPO.setTopicName(balanceTask.getTopic()); + ClusterBalanceReassignExtendData extendData = new ClusterBalanceReassignExtendData(); + extendData.setOriginalRetentionTimeUnitMs(topic.getRetentionMs()); + extendData.setReassignRetentionTimeUnitMs(topic.getRetentionMs()); + extendData.setOriginReplicaNum(topic.getReplicaNum()); + extendData.setReassignReplicaNum(balanceTask.getReplicas().size()); + reassignPO.setExtendData(ConvertUtil.obj2Json(extendData)); + reassignPO.setStatus(JobStatusEnum.WAITING.getStatus()); + return reassignPO; + } + + public static List convert2ListClusterBalanceReassignPO(List balanceTasks, Map topicMap, Long jobId, Long clusterId) { + List reassignPOs = new ArrayList<>(); + //生成迁移详情 + Map> balanceTaskMap = balanceTasks.stream().collect(Collectors.groupingBy(BalanceTask::getTopic)); + for (Map.Entry> entry : balanceTaskMap.entrySet()){ + Topic topic = topicMap.get(entry.getKey()); + if (topic == null || topic.getPartitionMap() == null){ + continue; + } + for (BalanceTask balanceTask : entry.getValue()){ + reassignPOs.add(ClusterBalanceConverter.convert2ClusterBalanceReassignPO(balanceTask, topic, jobId, clusterId)); + } + } + return reassignPOs; + } + + public static ClusterBalanceJobConfigPO convert2ClusterBalanceJobConfigPO(ClusterBalanceStrategyDTO dto, String operator) { + ClusterBalanceJobConfigPO jobConfigPO = new ClusterBalanceJobConfigPO(); + jobConfigPO.setCreator(operator); + jobConfigPO.setParallelNum(dto.getParallelNum()); + jobConfigPO.setThrottleUnitB(dto.getThrottleUnitB()); + jobConfigPO.setClusterId(dto.getClusterId()); + jobConfigPO.setExecutionStrategy(dto.getExecutionStrategy()); + jobConfigPO.setBalanceIntervalJson(ConvertUtil.obj2Json(dto.getClusterBalanceIntervalList())); + jobConfigPO.setTaskCron(dto.getScheduleCron()); + jobConfigPO.setMetricCalculationPeriod(dto.getMetricCalculationPeriod()); + jobConfigPO.setStatus(dto.getStatus()); + return jobConfigPO; + } + + public static JobClusterBalanceContent convert2JobClusterBalanceContent(ClusterBalanceJobConfigPO configPO) { + JobClusterBalanceContent content = new JobClusterBalanceContent(); + content.setType(JobTypeEnum.CLUSTER_BALANCE.getType()); + content.setParallelNum(configPO.getParallelNum()); + content.setThrottleUnitB(configPO.getThrottleUnitB()); + content.setClusterId(configPO.getClusterId()); + content.setExecutionStrategy(configPO.getExecutionStrategy()); + content.setClusterBalanceIntervalList(ConvertUtil.str2ObjArrayByJson(configPO.getBalanceIntervalJson(), ClusterBalanceIntervalDTO.class)); + content.setMetricCalculationPeriod(configPO.getMetricCalculationPeriod()); + content.setTopicBlackList(CommonUtils.string2StrList(configPO.getTopicBlackList())); + content.setScheduleJob(Boolean.TRUE); + return content; + } + + public static List convert2ClusterBalancePlanDetail(Map detailedMap) { + List details = new ArrayList<>(); + for(Map.Entry entry : detailedMap.entrySet()){ + BalanceDetailed balanceDetailed = entry.getValue(); + if (balanceDetailed == null){ + continue ; + } + ClusterBalancePlanDetail planDetail = new ClusterBalancePlanDetail(); + planDetail.setStatus(balanceDetailed.getBalanceState()==ClusterBalanceStateEnum.BALANCE.getState()?ClusterBalanceStateEnum.BALANCE.getState():ClusterBalanceStateEnum.UNBALANCED.getState()); + planDetail.setHost(balanceDetailed.getHost()); + planDetail.setBrokerId(entry.getKey()); + planDetail.setCpuBefore(balanceDetailed.getCurrentCPUUtilization()*Constant.ONE_HUNDRED); + planDetail.setCpuAfter(balanceDetailed.getLastCPUUtilization()*Constant.ONE_HUNDRED); + planDetail.setDiskBefore(balanceDetailed.getCurrentDiskUtilization()*Constant.ONE_HUNDRED); + planDetail.setDiskAfter(balanceDetailed.getLastDiskUtilization()*Constant.ONE_HUNDRED); + planDetail.setByteInBefore(balanceDetailed.getCurrentNetworkInUtilization()*Constant.ONE_HUNDRED); + planDetail.setByteInAfter(balanceDetailed.getLastNetworkInUtilization()*Constant.ONE_HUNDRED); + planDetail.setByteOutBefore(balanceDetailed.getCurrentNetworkOutUtilization()*Constant.ONE_HUNDRED); + planDetail.setByteOutAfter(balanceDetailed.getLastNetworkOutUtilization()*Constant.ONE_HUNDRED); + planDetail.setInReplica(balanceDetailed.getMoveInReplicas()); + planDetail.setOutReplica(balanceDetailed.getMoveOutReplicas()); + planDetail.setInSize(balanceDetailed.getMoveInDiskSize()); + planDetail.setOutSize(balanceDetailed.getMoveOutDiskSize()); + details.add(planDetail); + } + return details; + } + + //更新平衡任务完成后的集群均衡状态 + public static List convert2ClusterBalancePlanDetail(List details, Map stateMap) { + details.forEach(planDetail ->{ + BrokerBalanceState state = stateMap.get(planDetail.getBrokerId()); + if (state == null){ + return; + } + planDetail.setCpuStatus(state.getCpuBalanceState()); + planDetail.setDiskStatus(state.getDiskBalanceState()); + planDetail.setByteInStatus(state.getBytesInBalanceState()); + planDetail.setByteOutStatus(state.getBytesOutBalanceState()); + if ((state.getCpuBalanceState() == null || ClusterBalanceStateEnum.BALANCE.getState().equals(state.getCpuBalanceState())) + && (state.getDiskBalanceState() == null || ClusterBalanceStateEnum.BALANCE.getState().equals(state.getDiskBalanceState())) + && (state.getBytesInBalanceState() == null || ClusterBalanceStateEnum.BALANCE.getState().equals(state.getBytesInBalanceState())) + && (state.getBytesOutBalanceState() == null || ClusterBalanceStateEnum.BALANCE.getState().equals(state.getBytesOutBalanceState()))) { + planDetail.setStatus(ClusterBalanceStateEnum.BALANCE.getState()); + }else { + planDetail.setStatus(ClusterBalanceStateEnum.UNBALANCED.getState()); + } + }); + return details; + } + + public static List convert2ClusterBalancePlanDetailVO(List balanceBrokerIds, Map detailedMap) { + List detailVOS = new ArrayList<>(); + for(Map.Entry entry : detailedMap.entrySet()){ + BalanceDetailed value = entry.getValue(); + if (value == null || !balanceBrokerIds.contains(entry.getKey())){ + continue ; + } + ClusterBalancePlanDetailVO planDetailVO = new ClusterBalancePlanDetailVO(); + planDetailVO.setStatus(value.getBalanceState()==ClusterBalanceStateEnum.BALANCE.getState()?ClusterBalanceStateEnum.BALANCE.getState():ClusterBalanceStateEnum.UNBALANCED.getState()); + planDetailVO.setHost(value.getHost()); + planDetailVO.setBrokerId(entry.getKey()); + planDetailVO.setCpuBefore(value.getCurrentCPUUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setCpuAfter(value.getLastCPUUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setDiskBefore(value.getCurrentDiskUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setDiskAfter(value.getLastDiskUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setByteInBefore(value.getCurrentNetworkInUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setByteInAfter(value.getLastNetworkInUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setByteOutBefore(value.getCurrentNetworkOutUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setByteOutAfter(value.getLastNetworkOutUtilization()*Constant.ONE_HUNDRED); + planDetailVO.setInReplica(value.getMoveInReplicas()); + planDetailVO.setOutReplica(value.getMoveOutReplicas()); + planDetailVO.setInSize(value.getMoveInDiskSize()); + planDetailVO.setOutSize(value.getMoveOutDiskSize()); + detailVOS.add(planDetailVO); + } + return detailVOS; + } + + public static ClusterBalancePlanVO convert2ClusterBalancePlanVO(ClusterBalancePreviewDTO jobDTO, OptimizerResult optimizerResult, List allBrokers) { + if (ValidateUtils.anyNull(jobDTO, optimizerResult, optimizerResult.resultJsonOverview(), + optimizerResult.resultJsonDetailed(), optimizerResult.resultDetailed(), optimizerResult.resultJsonTask())){ + return null; + } + ClusterBalancePlanVO planVO = new ClusterBalancePlanVO(); + planVO.setTopics(CommonUtils.string2StrList(optimizerResult.resultOverview().getMoveTopics())); + planVO.setType(ClusterBalanceTypeEnum.IMMEDIATELY.getType()); + planVO.setReplicas(optimizerResult.resultOverview().getMoveReplicas()); + planVO.setBlackTopics(jobDTO.getTopicBlackList()); + planVO.setMoveSize(optimizerResult.resultOverview().getTotalMoveSize()); + planVO.setThreshold(ConvertUtil.obj2Json(jobDTO.getClusterBalanceIntervalList())); + planVO.setBrokers(convert2HostList(allBrokers, optimizerResult.resultOverview().getNodeRange())); + planVO.setDetail(convert2ClusterBalancePlanDetailVO(jobDTO.getBrokers(), optimizerResult.resultDetailed())); + planVO.setClusterBalanceIntervalList(ConvertUtil.list2List(jobDTO.getClusterBalanceIntervalList(), ClusterBalanceIntervalVO.class)); + planVO.setReassignmentJson(optimizerResult.resultJsonTask()); + return planVO; + } + + public static ClusterBalancePreviewDTO convert2ClusterBalancePreviewDTO(ClusterBalanceJobPO clusterBalanceJobPO) { + + ClusterBalancePreviewDTO planVO = new ClusterBalancePreviewDTO(); + planVO.setBrokers(CommonUtils.string2IntList(clusterBalanceJobPO.getBrokers())); + planVO.setClusterBalanceIntervalList(ConvertUtil.str2ObjArrayByJson(clusterBalanceJobPO.getBalanceIntervalJson(), ClusterBalanceIntervalDTO.class)); + planVO.setClusterId(clusterBalanceJobPO.getClusterId()); + planVO.setExecutionStrategy(clusterBalanceJobPO.getExecutionStrategy()); + planVO.setParallelNum(clusterBalanceJobPO.getParallelNum()); + planVO.setThrottleUnitB(clusterBalanceJobPO.getThrottleUnitB()); + planVO.setMetricCalculationPeriod(clusterBalanceJobPO.getMetricCalculationPeriod()); + planVO.setTopicBlackList(CommonUtils.string2StrList(clusterBalanceJobPO.getTopicBlackList())); + return planVO; + } + + public static Map convert2MapClusterBalanceOverviewSubVO(BrokerSpec brokerSpec, BrokerBalanceState state) { + Map subVOMap = new HashMap<>(); + if (brokerSpec == null){ + brokerSpec = new BrokerSpec(); + } + if (state == null){ + state = new BrokerBalanceState(); + } + Double cpuSpec = brokerSpec.getCpu()!=null?brokerSpec.getCpu()*Constant.ONE_HUNDRED:null;//转成基础单位 + subVOMap.put(Resource.DISK.resource(), + new ClusterBalanceOverviewSubVO( + state.getDiskAvgResource(), brokerSpec.getDisk(), + state.getDiskBalanceState() == null || state.getDiskBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())?state.getDiskBalanceState():ClusterBalanceStateEnum.UNBALANCED.getState())); + subVOMap.put(Resource.CPU.resource(), + new ClusterBalanceOverviewSubVO(state.getCpuAvgResource(), cpuSpec, + state.getCpuBalanceState() == null || state.getCpuBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())?state.getCpuBalanceState():ClusterBalanceStateEnum.UNBALANCED.getState())); + subVOMap.put(Resource.NW_IN.resource(), + new ClusterBalanceOverviewSubVO( + state.getBytesInAvgResource(), brokerSpec.getFlow(), + state.getBytesInBalanceState() == null || state.getBytesInBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())?state.getBytesInBalanceState():ClusterBalanceStateEnum.UNBALANCED.getState())); + subVOMap.put(Resource.NW_OUT.resource(), + new ClusterBalanceOverviewSubVO( + state.getBytesOutAvgResource(), brokerSpec.getFlow(), + state.getBytesOutBalanceState() == null || state.getBytesOutBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())?state.getBytesOutBalanceState():ClusterBalanceStateEnum.UNBALANCED.getState())); + return subVOMap; + } + + + + public static ClusterBalanceJobConfigVO convert2ClusterBalanceJobConfigVO(ClusterBalanceJobConfigPO clusterBalanceJobConfigPO){ + ClusterBalanceJobConfigVO configVO = new ClusterBalanceJobConfigVO(); + configVO.setScheduleCron(clusterBalanceJobConfigPO.getTaskCron()); + configVO.setClusterBalanceIntervalList(ConvertUtil.str2ObjArrayByJson(clusterBalanceJobConfigPO.getBalanceIntervalJson(), ClusterBalanceInterval.class)); + configVO.setClusterId(clusterBalanceJobConfigPO.getClusterId()); + configVO.setExecutionStrategy(clusterBalanceJobConfigPO.getExecutionStrategy()); + configVO.setParallelNum(clusterBalanceJobConfigPO.getParallelNum()); + configVO.setMetricCalculationPeriod(clusterBalanceJobConfigPO.getMetricCalculationPeriod()); + configVO.setThrottleUnitB(clusterBalanceJobConfigPO.getThrottleUnitB()); + configVO.setTopicBlackList(CommonUtils.string2StrList(clusterBalanceJobConfigPO.getTopicBlackList())); + configVO.setStatus(clusterBalanceJobConfigPO.getStatus()); + return configVO; + } + + + public static List convert2HostList(List allBrokers, String brokerIdStr){ + if (allBrokers.isEmpty() || ValidateUtils.isBlank(brokerIdStr)){ + return new ArrayList<>(); + } + List brokerIds = CommonUtils.string2IntList(brokerIdStr); + return allBrokers.stream().filter(broker -> brokerIds.contains(broker.getBrokerId())) + .map(Broker::getHost).collect(Collectors.toList()); + } + + private static List convert2ListHostEnv(Map brokerMap, Map brokerSpecMap) { + List hostEnvs = new ArrayList<>(); + for (Map.Entry entry : brokerMap.entrySet()) { + HostEnv hostEnv = new HostEnv(); + hostEnv.setId(entry.getKey()); + hostEnv.setHost(entry.getValue().getHost()); + hostEnv.setRackId(entry.getValue().getRack()); + BrokerSpec brokerSpec = brokerSpecMap.get(entry.getKey()); + if (brokerSpec == null){ + continue; + } + hostEnv.setCpu(brokerSpec.getCpu().intValue() * Constant.ONE_HUNDRED); + hostEnv.setDisk(brokerSpec.getDisk() * Constant.B_TO_GB); + hostEnv.setNetwork(brokerSpec.getFlow() * Constant.B_TO_MB); + hostEnvs.add(hostEnv); + } + + return hostEnvs; + + } + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceReassignConverter.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceReassignConverter.java new file mode 100644 index 00000000..0bdb4d73 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/converter/ClusterBalanceReassignConverter.java @@ -0,0 +1,218 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.ClusterBalanceReassignDetail; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.ClusterBalanceReassignExtendData; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail.ClusterBalanceDetailDataGroupByPartition; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail.ClusterBalanceDetailDataGroupByTopic; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.JobStatus; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobDetail; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.SubJobReplicaMoveDetail; +import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReplaceReassignSub; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobClusterBalanceReplicaMoveVO; +import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobPartitionDetailVO; +import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobVO; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum; +import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; + +import java.util.*; +import java.util.stream.Collectors; + +@EnterpriseLoadReBalance +public class ClusterBalanceReassignConverter { + + private ClusterBalanceReassignConverter() { + } + + public static JobDetail convert2JobDetail(Job job, ClusterBalanceReassignDetail reassignDetail) { + JobDetail jobDetail = new JobDetail(); + jobDetail.setId(job.getId()); + jobDetail.setJobType(job.getJobType()); + jobDetail.setJobName(job.getJobName()); + jobDetail.setJobStatus(job.getJobStatus()); + jobDetail.setPlanTime(job.getPlanTime()); + + jobDetail.setStartTime(reassignDetail.getStartTime()); + jobDetail.setEndTime(reassignDetail.getFinishedTime()); + jobDetail.setFlowLimit(reassignDetail.getThrottleUnitB().doubleValue()); + + JobStatus jobStatus = new JobStatus(reassignDetail.getReassignTopicDetailsList().stream().map(elem -> elem.getStatus()).collect(Collectors.toList())); + jobDetail.setTotal(jobStatus.getTotal()); + jobDetail.setSuccess(jobStatus.getSuccess()); + jobDetail.setFail(jobStatus.getFailed()); + jobDetail.setDoing(jobStatus.getDoing()); + + List subJobDetailList = new ArrayList<>(); + subJobDetailList.addAll( + ConvertUtil.list2List(convert2SubJobReplicaMoveDetailList(reassignDetail.getReassignTopicDetailsList()), SubJobClusterBalanceReplicaMoveVO.class) + ); + jobDetail.setSubJobs(subJobDetailList); + + return jobDetail; + } + + public static ClusterBalanceReassignDetail convert2ClusterBalanceReassignDetail(ClusterBalanceJobPO jobPO, List reassignPOS) { + // 按照Topic做聚合 + Map> topicJobPOMap = new HashMap<>(); + reassignPOS.forEach(elem -> { + topicJobPOMap.putIfAbsent(elem.getTopicName(), new ArrayList<>()); + topicJobPOMap.get(elem.getTopicName()).add(elem); + }); + + List reassignTopicDetailsList = new ArrayList<>(); + for (Map.Entry> entry: topicJobPOMap.entrySet()) { + reassignTopicDetailsList.add(convert2ClusterBalanceDetailDataGroupByTopic(entry.getValue())); + } + + ClusterBalanceReassignDetail jobDetail = new ClusterBalanceReassignDetail(); + jobDetail.setThrottleUnitB(jobPO.getThrottleUnitB()); + jobDetail.setReassignTopicDetailsList(reassignTopicDetailsList); + jobDetail.setStartTime(jobPO.getStartTime()); + if (JobStatusEnum.isFinished(jobPO.getStatus())) { + jobDetail.setFinishedTime(jobPO.getFinishedTime()); + } + return jobDetail; + } + + private static ClusterBalanceDetailDataGroupByTopic convert2ClusterBalanceDetailDataGroupByTopic(List reassingns) { + Set originalBrokerIdSet = new HashSet<>(); + Set reassignBrokerIdSet = new HashSet<>(); + + // 分区的信息 + List partitionDetailList = new ArrayList<>(); + for (ClusterBalanceReassignPO reassignPO : reassingns) { + ClusterBalanceDetailDataGroupByPartition detail = new ClusterBalanceDetailDataGroupByPartition(); + detail.setPartitionId(reassignPO.getPartitionId()); + detail.setClusterPhyId(reassignPO.getClusterId()); + detail.setTopicName(reassignPO.getTopicName()); + detail.setOriginalBrokerIdList(CommonUtils.string2IntList(reassignPO.getOriginalBrokerIds())); + detail.setReassignBrokerIdList(CommonUtils.string2IntList(reassignPO.getReassignBrokerIds())); + detail.setStatus(reassignPO.getStatus()); + + ClusterBalanceReassignExtendData extendData = ConvertUtil.str2ObjByJson(reassignPO.getExtendData(), ClusterBalanceReassignExtendData.class); + if (extendData != null) { + detail.setNeedReassignLogSizeUnitB(extendData.getNeedReassignLogSizeUnitB()); + detail.setFinishedReassignLogSizeUnitB(extendData.getFinishedReassignLogSizeUnitB()); + detail.setRemainTimeUnitMs(extendData.getRemainTimeUnitMs()); + detail.setPresentReplicaNum(extendData.getOriginReplicaNum()); + detail.setNewReplicaNum(extendData.getReassignReplicaNum()); + detail.setOriginalRetentionTimeUnitMs(extendData.getOriginalRetentionTimeUnitMs()); + detail.setReassignRetentionTimeUnitMs(extendData.getReassignRetentionTimeUnitMs()); + } + + originalBrokerIdSet.addAll(detail.getOriginalBrokerIdList()); + reassignBrokerIdSet.addAll(detail.getReassignBrokerIdList()); + partitionDetailList.add(detail); + } + // Topic的详细信息 + ClusterBalanceDetailDataGroupByTopic topicDetail = new ClusterBalanceDetailDataGroupByTopic(); + topicDetail.setPartitionIdList(partitionDetailList.stream().map(elem -> elem.getPartitionId()).collect(Collectors.toList())); + topicDetail.setReassignPartitionDetailsList(partitionDetailList); + topicDetail.setClusterPhyId(reassingns.get(0).getClusterId()); + topicDetail.setTopicName(reassingns.get(0).getTopicName()); + + topicDetail.setOriginalBrokerIdList(new ArrayList<>(originalBrokerIdSet)); + topicDetail.setReassignBrokerIdList(new ArrayList<>(reassignBrokerIdSet)); + + List needSizeList = partitionDetailList + .stream() + .filter(elem -> elem.getNeedReassignLogSizeUnitB() != null) + .map(item -> item.getNeedReassignLogSizeUnitB()).collect(Collectors.toList()); + topicDetail.setNeedReassignLogSizeUnitB(needSizeList.isEmpty()? null: needSizeList.stream().reduce(Long::sum).get()); + + List finishedSizeList = partitionDetailList + .stream() + .filter(elem -> elem.getFinishedReassignLogSizeUnitB() != null) + .map(item -> item.getFinishedReassignLogSizeUnitB()).collect(Collectors.toList()); + topicDetail.setFinishedReassignLogSizeUnitB(finishedSizeList.isEmpty()? null: finishedSizeList.stream().reduce(Long::sum).get()); + + List remainList = partitionDetailList + .stream() + .filter(elem -> elem.getRemainTimeUnitMs() != null) + .map(item -> item.getRemainTimeUnitMs()).collect(Collectors.toList()); + topicDetail.setRemainTimeUnitMs(remainList.isEmpty()? null: remainList.stream().reduce(Long::max).get()); + + topicDetail.setPresentReplicaNum(partitionDetailList.get(0).getPresentReplicaNum()); + topicDetail.setNewReplicaNum(partitionDetailList.get(0).getNewReplicaNum()); + topicDetail.setOriginalRetentionTimeUnitMs(partitionDetailList.get(0).getOriginalRetentionTimeUnitMs()); + topicDetail.setReassignRetentionTimeUnitMs(partitionDetailList.get(0).getReassignRetentionTimeUnitMs()); + + topicDetail.setStatus( + new JobStatus( + partitionDetailList.stream().map(elem -> elem.getStatus()).collect(Collectors.toList()) + ).getStatus() + ); + + return topicDetail; + } + + public static List convert2SubJobPartitionDetailVOList(ClusterBalanceDetailDataGroupByTopic detailDataGroupByTopic) { + List voList = new ArrayList<>(); + for (ClusterBalanceDetailDataGroupByPartition groupByPartition: detailDataGroupByTopic.getReassignPartitionDetailsList()) { + SubJobPartitionDetailVO vo = new SubJobPartitionDetailVO(); + vo.setPartitionId(groupByPartition.getPartitionId()); + vo.setSourceBrokerIds(groupByPartition.getOriginalBrokerIdList()); + vo.setDesBrokerIds(groupByPartition.getReassignBrokerIdList()); + vo.setTotalSize(groupByPartition.getNeedReassignLogSizeUnitB() != null ? groupByPartition.getNeedReassignLogSizeUnitB().doubleValue(): null); + vo.setMovedSize(groupByPartition.getFinishedReassignLogSizeUnitB() != null ? groupByPartition.getFinishedReassignLogSizeUnitB().doubleValue(): null); + vo.setStatus(groupByPartition.getStatus()); + vo.setRemainTime(groupByPartition.getRemainTimeUnitMs()); + + voList.add(vo); + } + + return voList; + } + + private static List convert2SubJobReplicaMoveDetailList(List reassignTopicDetailsList) { + List detailList = new ArrayList<>(); + + for (ClusterBalanceDetailDataGroupByTopic detailDataGroupByTopic: reassignTopicDetailsList) { + SubJobReplicaMoveDetail detail = new SubJobReplicaMoveDetail(); + detail.setTopicName(detailDataGroupByTopic.getTopicName()); + detail.setPartitions(detailDataGroupByTopic.getPartitionIdList()); + detail.setCurrentTimeSpent(detailDataGroupByTopic.getOriginalRetentionTimeUnitMs()); + detail.setMoveTimeSpent(detailDataGroupByTopic.getReassignRetentionTimeUnitMs()); + detail.setSourceBrokers(detailDataGroupByTopic.getOriginalBrokerIdList()); + detail.setDesBrokers(detailDataGroupByTopic.getReassignBrokerIdList()); + detail.setStatus(detailDataGroupByTopic.getStatus()); + if (detailDataGroupByTopic.getNeedReassignLogSizeUnitB() != null) { + detail.setTotalSize(detailDataGroupByTopic.getNeedReassignLogSizeUnitB().doubleValue()); + } + if (detailDataGroupByTopic.getFinishedReassignLogSizeUnitB() != null) { + detail.setMovedSize(detailDataGroupByTopic.getFinishedReassignLogSizeUnitB().doubleValue()); + } + JobStatus jobStatus = new JobStatus(detailDataGroupByTopic.getReassignPartitionDetailsList().stream().map(elem -> elem.getStatus()).collect(Collectors.toList())); detail.setTotal(jobStatus.getTotal()); + detail.setSuccess(jobStatus.getSuccess()); + detail.setFail(jobStatus.getFailed()); + detail.setDoing(jobStatus.getDoing()); + detail.setRemainTime(detailDataGroupByTopic.getRemainTimeUnitMs()); + detailList.add(detail); + } + + return detailList; + } + + public static List convert2ReplaceReassignSubList(List reassignPOList) { + List voList = new ArrayList<>(); + for (ClusterBalanceReassignPO reassignPO: reassignPOList) { + voList.add(convert2ReplaceReassignSub(reassignPO)); + } + return voList; + } + + public static ReplaceReassignSub convert2ReplaceReassignSub(ClusterBalanceReassignPO reassignPO) { + ReplaceReassignSub reassignSub = new ReplaceReassignSub(); + reassignSub.setClusterPhyId(reassignPO.getClusterId()); + reassignSub.setOriginalBrokerIdList(CommonUtils.string2IntList(reassignPO.getOriginalBrokerIds())); + reassignSub.setReassignBrokerIdList(CommonUtils.string2IntList(reassignPO.getReassignBrokerIds())); + reassignSub.setPartitionId(reassignPO.getPartitionId()); + reassignSub.setTopicName(reassignPO.getTopicName()); + return reassignSub; + } + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceStateEnum.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceStateEnum.java new file mode 100644 index 00000000..5a2036c5 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceStateEnum.java @@ -0,0 +1,31 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.enums; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Getter; + +/** + * 集群平衡状态 + * @author zengqiao + * @date 22/03/08 + */ +@Getter +@EnterpriseLoadReBalance +public enum ClusterBalanceStateEnum { + BELOW_BALANCE(-1, "低于均衡范围"), + + BALANCE(0, "均衡范围内"), + + ABOVE_BALANCE(1, "高于均衡范围"), + + UNBALANCED(2, "不均衡"), + ; + + private final Integer state; + + private final String message; + + ClusterBalanceStateEnum(int state, String message) { + this.state = state; + this.message = message; + } +} \ No newline at end of file diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceTypeEnum.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceTypeEnum.java new file mode 100644 index 00000000..7e189c6a --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/enums/ClusterBalanceTypeEnum.java @@ -0,0 +1,28 @@ +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.enums; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import lombok.Getter; + +/** + * 集群平衡任务类型 + * @author zengqiao + * @date 22/03/08 + */ +@Getter +@EnterpriseLoadReBalance +public enum ClusterBalanceTypeEnum { + + IMMEDIATELY(1, "立即"), + + CYCLE(2, "周期"), + ; + + private final int type; + + private final String message; + + ClusterBalanceTypeEnum(int type, String message) { + this.type = type; + this.message = message; + } +} \ No newline at end of file diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/package-info.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/package-info.java new file mode 100644 index 00000000..019215c4 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enterprise/rebalance/package-info.java @@ -0,0 +1,9 @@ +/** + * Load-reBalance相关功能模块 + * km-extends/km-rebalance 模块，是依据指标生成迁移 plan 的模块，是底层的一个基础功能 + * 当前 package 模块是依据产品的要求，依赖 km-extends/km-rebalance 模块，构建产品实际使用功能 + */ +@EnterpriseLoadReBalance +package com.xiaojukeji.know.streaming.km.common.enterprise.rebalance; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/ModuleEnum.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/ModuleEnum.java index 7c07718b..0e2b6abb 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/ModuleEnum.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/ModuleEnum.java @@ -2,6 +2,7 @@ package com.xiaojukeji.know.streaming.km.common.enums.operaterecord; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; import com.xiaojukeji.know.streaming.km.common.constant.Constant; import java.util.List; @@ -37,6 +38,9 @@ public enum ModuleEnum { JOB_KAFKA_REPLICA_REASSIGN(110, "Job-KafkaReplica迁移"), + @EnterpriseLoadReBalance + JOB_CLUSTER_BALANCE(111, "Job-ClusterBalance"), + ; ModuleEnum(int code, String desc) { diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/job/ClusterBalanceJobHandler.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/job/ClusterBalanceJobHandler.java new file mode 100644 index 00000000..21b2f9c6 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/job/ClusterBalanceJobHandler.java @@ -0,0 +1,355 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.job; + +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec; +import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.ClusterBalanceReassignDetail; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail.ClusterBalanceDetailDataGroupByTopic; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.JobStatus; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.content.JobClusterBalanceContent; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobDetail; +import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobModifyDetail; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus; +import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import com.xiaojukeji.know.streaming.km.common.bean.po.job.JobPO; +import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobPartitionDetailVO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.common.constant.JobConstant; +import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter.ClusterBalanceConverter; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter.ClusterBalanceReassignConverter; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobActionEnum; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService; +import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceReassignService; +import com.xiaojukeji.know.streaming.km.core.service.config.ConfigUtils; +import com.xiaojukeji.know.streaming.km.core.service.job.JobHandler; +import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; +import com.xiaojukeji.know.streaming.km.persistence.mysql.job.JobDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.interceptor.TransactionAspectSupport; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@EnterpriseLoadReBalance +@Component(JobConstant.CLUSTER_BALANCE) +public class ClusterBalanceJobHandler implements JobHandler { + + private static final ILog logger = LogFactory.getLog(ClusterBalanceJobHandler.class); + + @Value("${es.client.address:}") + private String esAddress; + + @Autowired + private ClusterBalanceJobService clusterBalanceJobService; + + @Autowired + private ClusterPhyService clusterPhyService; + + @Autowired + private ClusterBalanceReassignService clusterBalanceReassignService; + + @Autowired + private KafkaZKDAO kafkaZKDAO; + + @Autowired + private JobDAO jobDAO; + + @Autowired + private BrokerService brokerService; + + @Autowired + private BrokerSpecService brokerSpecService; + + @Autowired + private TopicService topicService; + + @Autowired + private ConfigUtils configUtils; + + @Override + public JobTypeEnum type() { + return JobTypeEnum.CLUSTER_BALANCE; + } + + @Override + @Transactional + public Result submit(Job job, String operator) { + // 获取任务详情信息 + JobClusterBalanceContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobClusterBalanceContent.class); + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterId()); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + //获取broke规格信息 + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhy.getId()); + //获取集群所有broker信息 + List brokers = brokerService.listAllBrokersFromDB(clusterPhy.getId()); + for(Broker broker:brokers){ + if (brokerSpecMap.get(broker.getBrokerId()) == null){ + return Result.buildFromRSAndMsg(ResultStatus.BROKER_SPEC_NOT_EXIST,String.format("Broker规格信息不存在:brokerId:%s", broker.getBrokerId())); + } + } + + //获取任务计划 + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(dto.getClusterId(), configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + BalanceParameter balanceParameter = ClusterBalanceConverter.convert2BalanceParameter(dto, brokers, brokerSpecMap, clusterPhy, esAddress, topicNames); + try { + ExecutionRebalance executionRebalance = new ExecutionRebalance(); + OptimizerResult optimizerResult = executionRebalance.optimizations(balanceParameter); + Result cRs = checkOptimizerResult(optimizerResult, job.getId()); + if (cRs.failed()){ + return cRs; + } + + Map topicMap = topicService.listTopicsFromDB(clusterPhy.getId()).stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity())); + List reassignPOS = ClusterBalanceConverter.convert2ListClusterBalanceReassignPO( + optimizerResult.resultTask(), topicMap, job.getId(), clusterPhy.getId()); + + String generateReassignmentJson = optimizerResult.resultJsonTask(); + if (dto.getParallelNum() > 0){ + //根据执行策略生成迁移json + Result jResult = clusterBalanceJobService.generateReassignmentJson(job.getClusterId(),dto.getParallelNum(), dto.getExecutionStrategy(), Constant.NUM_ONE, reassignPOS); + if (jResult.failed()){ + return Result.buildFromIgnoreData(jResult); + } + generateReassignmentJson = jResult.getData(); + } + + //生成平衡job + ClusterBalanceJobPO clusterBalanceJobPO = ClusterBalanceConverter.convert2ClusterBalanceJobPO(job.getId(), dto, optimizerResult, brokers, operator, generateReassignmentJson); + Result result = clusterBalanceJobService.createClusterBalanceJob(clusterBalanceJobPO, operator); + if (result.failed()){ + logger.error("method=clusterBalanceJobHandler.submit||job={}||errMsg={}!", + job, result.getMessage()); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return result; + } + + //生成迁移明细 + Result cbrResult = clusterBalanceReassignService.addBatchBalanceReassign(reassignPOS); + if (cbrResult.failed()){ + logger.error("method=clusterBalanceJobHandler.submit||job={}||errMsg={}!", + job, cbrResult.getMessage()); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return cbrResult; + } + + //更新job执行对象 + job.setTarget(optimizerResult.resultOverview().getMoveTopics()); + int count = jobDAO.updateById(ConvertUtil.obj2Obj(job, JobPO.class)); + if (count < 0){ + logger.error("method=clusterBalanceJobHandler.submit||job={}||errMsg={}!", + job, result.getMessage()); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + + }catch (Exception e){ + logger.error("method=clusterBalanceJobHandler.submit||job={}||errMsg=exception", job, e); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFailure(e.getMessage()); + } + return Result.buildSuc(); + } + + @Override + @Transactional + public Result delete(Job job, String operator) { + //删除balanceJob + Result balanceJobResult = clusterBalanceJobService.deleteByJobId(job.getId(), operator); + if (balanceJobResult.failed()){ + logger.error("method=clusterBalanceJobHandler.delete||job={}||operator:{}||errMsg={}", job, operator, balanceJobResult); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return balanceJobResult; + } + return Result.buildSuc(); + } + + @Override + public Result modify(Job job, String operator) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(job.getClusterId()); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + Result balanceJobPOResult = clusterBalanceJobService.getClusterBalanceJobById(job.getId()); + if (!balanceJobPOResult.hasData()){ + return Result.buildFrom(ResultStatus.NOT_EXIST); + } + + List brokers = brokerService.listAllBrokersFromDB(clusterPhy.getId()); + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhy.getId()); + + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(job.getClusterId(), configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + JobClusterBalanceContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobClusterBalanceContent.class); + BalanceParameter balanceParameter = ClusterBalanceConverter.convert2BalanceParameter(dto, brokers, brokerSpecMap, clusterPhy, esAddress, topicNames); + ExecutionRebalance executionRebalance = new ExecutionRebalance(); + try { + OptimizerResult optimizerResult = executionRebalance.optimizations(balanceParameter); + Result cRs = checkOptimizerResult(optimizerResult, job.getId()); + if (cRs.failed()){ + return cRs; + } + + Map topicMap = kafkaZKDAO.getAllTopicMetadata(clusterPhy.getId(), false).stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity())); + List reassignPOS = ClusterBalanceConverter.convert2ListClusterBalanceReassignPO(optimizerResult.resultTask(), + topicMap, job.getId(), clusterPhy.getId()); + + String generateReassignmentJson = optimizerResult.resultJsonTask(); + if (dto.getParallelNum() > 0){ + //根据执行策略生成迁移json + Result jResult = clusterBalanceJobService.generateReassignmentJson(job.getClusterId(),dto.getParallelNum(), dto.getExecutionStrategy(), Constant.NUM_ONE, reassignPOS); + if (jResult.failed()){ + return Result.buildFromIgnoreData(jResult); + } + generateReassignmentJson = jResult.getData(); + } + //生成平衡job + ClusterBalanceJobPO clusterBalanceJobPO = ClusterBalanceConverter.convert2ClusterBalanceJobPO(job.getId(), dto ,optimizerResult, brokers, operator, generateReassignmentJson); + Result result = clusterBalanceJobService.modifyClusterBalanceJob(clusterBalanceJobPO, operator); + if (result.failed()){ + return result; + } + + //删除原迁移详情，生成新的迁移详情 + Result delReassignResult = clusterBalanceReassignService.delete(job.getId(), operator); + if (delReassignResult.failed()){ + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return delReassignResult; + } + Result cbrResult = clusterBalanceReassignService.addBatchBalanceReassign(reassignPOS); + if (cbrResult.failed()){ + logger.error("method=clusterBalanceJobHandler.submit||job={}||errMsg={}!", + job, cbrResult.getMessage()); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return cbrResult; + } + }catch (Exception e){ + logger.error("method=clusterBalanceJobHandler.modify||job={}||operator:{}||errMsg=exception", job, operator, e); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage()); + } + return Result.buildSuc(); + } + + @Override + public Result updateLimit(Job job, Long limit, String operator) { + return clusterBalanceReassignService.modifyThrottle(job.getId(), limit, operator); + } + + @Override + public Result process(Job job, JobActionEnum action, String operator) { + if (JobActionEnum.START.equals(action)) { + return clusterBalanceReassignService.execute(job.getId()); + } + + if (JobActionEnum.CANCEL.equals(action)) { + return clusterBalanceReassignService.cancel(job.getId()); + } + + // 迁移中，不支持该操作 + return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FORBIDDEN, String.format("不支持[%s]操作", action.getValue())); + } + + @Override + public Result status(Job job) { + // Topic下每个分区的状态 + Map> topicJobsMap = new HashMap<>(); + + // 获取子任务，并按照Topic进行聚合 + List allSubJobPOList = clusterBalanceReassignService.getBalanceReassignsByJobId(job.getId()); + allSubJobPOList.forEach(elem -> { + topicJobsMap.putIfAbsent(elem.getTopicName(), new ArrayList<>()); + topicJobsMap.get(elem.getTopicName()).add(elem); + }); + + // 获取每个Topic的状态 + List topicStatusList = new ArrayList<>(); + for (List topicJobPOList: topicJobsMap.values()) { + topicStatusList.add(new JobStatus( + topicJobPOList.stream().map(elem -> elem.getStatus()).collect(Collectors.toList()) + ).getStatus()); + } + + // 聚合Topic的结果 + return Result.buildSuc(new JobStatus(topicStatusList)); + } + + @Override + public Result getTaskDetail(Job job) { + Result detailResult = clusterBalanceReassignService.getJobDetailsGroupByTopic(job.getId()); + if (detailResult.failed()) { + return Result.buildFromIgnoreData(detailResult); + } + + return Result.buildSuc(ClusterBalanceReassignConverter.convert2JobDetail(job, detailResult.getData())); + } + + @Override + public Result getTaskModifyDetail(Job job) { + // 获取任务详情信息 + JobClusterBalanceContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobClusterBalanceContent.class); + if (dto == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "jobData格式错误"); + } + + JobModifyDetail detail = ConvertUtil.obj2Obj(job, JobModifyDetail.class); + detail.setJobData(ConvertUtil.obj2Json(dto)); + return Result.buildSuc(detail); + } + + @Override + public Result> getSubJobPartitionDetail(Job job, String topic) { + Result detailResult = clusterBalanceReassignService.getJobDetailsGroupByTopic(job.getId()); + if (detailResult.failed()) { + return Result.buildFromIgnoreData(detailResult); + } + + List detailDataGroupByTopicList = detailResult.getData().getReassignTopicDetailsList() + .stream() + .filter(elem -> elem.getTopicName().equals(topic)) + .collect(Collectors.toList()); + + if (detailDataGroupByTopicList.isEmpty()) { + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(job.getClusterId(), topic)); + } + + return Result.buildSuc(ClusterBalanceReassignConverter.convert2SubJobPartitionDetailVOList(detailDataGroupByTopicList.get(0))); + } + + + + private Result checkOptimizerResult(OptimizerResult optimizerResult, Long jobId){ + if (optimizerResult == null){ + return Result.buildFrom(ResultStatus.KAFKA_OPERATE_FAILED); + } + if (optimizerResult.resultOverview().getMoveReplicas() == 0){ + logger.info("method=checkOptimizerResult||jobId:{}||msg=the cluster has reached equilibrium", jobId); + return Result.buildFailure("该集群已达到均衡要求，不需要再执行均衡任务。"); + } + return Result.buildSuc(); + } +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/package-info.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/package-info.java new file mode 100644 index 00000000..fa0b61e6 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/package-info.java @@ -0,0 +1,9 @@ +/** + * Load-reBalance相关功能模块 + * km-extends/km-rebalance 模块，是依据指标生成迁移 plan 的模块，是底层的一个基础功能 + * 当前 package 模块是依据产品的要求，依赖 km-extends/km-rebalance 模块，构建产品实际使用功能 + */ +@EnterpriseLoadReBalance +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobConfigService.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobConfigService.java new file mode 100644 index 00000000..201ca57e --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobConfigService.java @@ -0,0 +1,23 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobConfigPO; + +@EnterpriseLoadReBalance +public interface ClusterBalanceJobConfigService { + + /** + * 新增平衡配置 + * @param clusterBalanceJobConfigPO + * @return + */ + Result replaceClusterBalanceJobConfigByClusterId(ClusterBalanceJobConfigPO clusterBalanceJobConfigPO); + + /** + * + * @param clusterId + * @return + */ + Result getByClusterId(Long clusterId); +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobService.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobService.java new file mode 100644 index 00000000..310b3fb9 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceJobService.java @@ -0,0 +1,93 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.ClusterBalanceHistoryVO; + +import java.util.List; +import java.util.Map; + +@EnterpriseLoadReBalance +public interface ClusterBalanceJobService { + /** + * + * @param jobId jobId + * @return + */ + Result deleteByJobId(Long jobId, String operator); + + /** + * + * @param clusterBalanceJobPO + * @return + */ + Result createClusterBalanceJob(ClusterBalanceJobPO clusterBalanceJobPO, String operator); + + /** + * + * @param clusterBalanceJobPO + * @return + */ + Result modifyClusterBalanceJob(ClusterBalanceJobPO clusterBalanceJobPO, String operator); + + /** + * + * @param id id + * @return + */ + Result getClusterBalanceJobById(Long id); + + /** + * + * @param clusterPhyId + * @return + */ + ClusterBalanceJobPO getLastOneByClusterId(Long clusterPhyId); + + /** + * + * @param clusterPhyId + * @return + */ + Map getBalanceInterval(Long clusterPhyId); + + /** + * + * @param clusterPhyId + * @return + */ + PaginationResult page(Long clusterPhyId, PaginationBaseDTO dto); + + /** + * 依据任务状态或者其中一个任务ID + */ + Long getOneRunningJob(Long clusterPhyId); + + /** + * 检查平衡任务 + */ + Result verifyClusterBalanceAndUpdateStatue(Long jobId); + + /** + * 根据jobId生成迁移json + * @param parallelNum 并行数 + * @param clusterId 集群id + * @param executionStrategy 执行策略 + * @param reassignPOList 迁移任务详情 + * @return + */ + Result generateReassignmentJson(Long clusterId, Integer parallelNum, Integer jsonVersion, Integer executionStrategy, List reassignPOList); + + /** + * 根据迁移策略更新迁移任务 + * @param jobId jobId + * @param clusterPhyId 集群id + * @return + */ + Result generateReassignmentForStrategy(Long clusterPhyId, Long jobId); + +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceReassignService.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceReassignService.java new file mode 100644 index 00000000..4f8c6301 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceReassignService.java @@ -0,0 +1,72 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.ClusterBalanceReassignDetail; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; + +import java.util.List; + +@EnterpriseLoadReBalance +public interface ClusterBalanceReassignService { + + /** + *新增迁移任务 + * @param clusterBalanceReassignPO + * @return + */ + Result addBalanceReassign(ClusterBalanceReassignPO clusterBalanceReassignPO); + + /** + *批量新增迁移任务 + * @param reassignPOList + * @return + */ + Result addBatchBalanceReassign(List reassignPOList); + /** + * 删除迁移任务 + */ + Result delete(Long jobId, String operator); + + /** + * 执行迁移任务 + */ + Result execute(Long jobId); + + /** + * 取消迁移任务 + */ + Result cancel(Long jobId); + + /** + * 检查迁移任务 + */ + Result verifyAndUpdateStatue(ClusterBalanceJobPO clusterBalanceJobPO); + + /** + * 修改限流值 + */ + Result modifyThrottle(Long jobId, Long throttleUnitB, String operator); + + /** + * 更新子任务中扩展字段的数据 + */ + Result getAndUpdateSubJobExtendData(Long jobId); + + /** + * 获取迁移任务信息 + */ + List getBalanceReassignsByJobId(Long jobId); + + /** + * 获取按照Topic维度聚合的详情 + */ + Result getJobDetailsGroupByTopic(Long jobId); + + /** + * leader重新选举 + */ + Result preferredReplicaElection(Long jobId); + +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceService.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceService.java new file mode 100644 index 00000000..bfe2d6b1 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/ClusterBalanceService.java @@ -0,0 +1,82 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceOverviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalancePreviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceStrategyDTO; +import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.ClusterBalanceItemState; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.*; + +@EnterpriseLoadReBalance +public interface ClusterBalanceService { + + /** + * @param clusterPhyId + * @return + */ + Result state(Long clusterPhyId); + + /** + * @param clusterPhyId + * @return + */ + Result config(Long clusterPhyId); + + /** + * @param clusterPhyId + * @param dto + * @return + */ + PaginationResult overview(Long clusterPhyId, ClusterBalanceOverviewDTO dto); + + /** + * @param clusterPhyId + * @return + */ + Result getItemState(Long clusterPhyId); + + /** + * @param clusterPhyId + * @param dto + * @return + */ + PaginationResult history(Long clusterPhyId, PaginationBaseDTO dto); + + /** + * @param clusterPhyId + * @param jobId + * @return + */ + Result plan(Long clusterPhyId, Long jobId); + + + /** + * @param clusterBalancePreviewDTO + * @return + */ + Result preview(Long clusterPhyId, ClusterBalancePreviewDTO clusterBalancePreviewDTO); + + /** + * @param jobId + * @return + */ + Result schedule(Long clusterPhyId, Long jobId); + + /** + * @param clusterPhyId + * @param dto + * @return + */ + Result strategy(Long clusterPhyId, ClusterBalanceStrategyDTO dto, String operator); + + + /** + * @param clusterPhyId + * @return + */ + Result createScheduleJob(Long clusterPhyId, long triggerTimeUnitMs); + +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobConfigServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobConfigServiceImpl.java new file mode 100644 index 00000000..abdae4c6 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobConfigServiceImpl.java @@ -0,0 +1,79 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobConfigPO; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobConfigService; +import com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance.ClusterBalanceJobConfigDao; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +@Service +@EnterpriseLoadReBalance +public class ClusterBalanceJobConfigServiceImpl implements ClusterBalanceJobConfigService { + private static final ILog logger = LogFactory.getLog(ClusterBalanceJobConfigServiceImpl.class); + + @Autowired + private ClusterBalanceJobConfigDao clusterBalanceJobConfigDao; + + @Autowired + private BrokerSpecService brokerSpecService; + + @Autowired + private BrokerService brokerService; + + @Override + public Result replaceClusterBalanceJobConfigByClusterId(ClusterBalanceJobConfigPO clusterBalanceJobConfigPO) { + List brokers = brokerService.listAllBrokersFromDB(clusterBalanceJobConfigPO.getClusterId()); + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterBalanceJobConfigPO.getClusterId()); + for(Broker broker: brokers){ + if (brokerSpecMap.get(broker.getBrokerId())==null){ + return Result.buildFrom(ResultStatus.CLUSTER_SPEC_INCOMPLETE); + } + } + + try { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper(); + queryWrapper.eq(ClusterBalanceJobConfigPO::getClusterId, clusterBalanceJobConfigPO.getClusterId()); + ClusterBalanceJobConfigPO oldConfig = clusterBalanceJobConfigDao.selectOne(queryWrapper); + + int count; + if (oldConfig == null){ + count = clusterBalanceJobConfigDao.insert(clusterBalanceJobConfigPO); + + }else{ + clusterBalanceJobConfigPO.setId(oldConfig.getId()); + count = clusterBalanceJobConfigDao.updateById(clusterBalanceJobConfigPO); + } + if (count < 1){ + logger.error("replace cluster balance job config detail failed! clusterBalanceJobConfigPO:{}", clusterBalanceJobConfigPO); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + + }catch (Exception e){ + logger.error("replace cluster balance job config failed! clusterBalanceJobConfigPO:{}", clusterBalanceJobConfigPO, e); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + + } + return Result.buildSuc(); + } + + @Override + public Result getByClusterId(Long clusterId) { + ClusterBalanceJobConfigPO queryParam = new ClusterBalanceJobConfigPO(); + queryParam.setClusterId(clusterId); + return Result.buildSuc(clusterBalanceJobConfigDao.selectOne(new QueryWrapper<>(queryParam))); + } +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobServiceImpl.java new file mode 100644 index 00000000..13f961a3 --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceJobServiceImpl.java @@ -0,0 +1,458 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.impl; + +import com.alibaba.fastjson.JSON; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.google.common.collect.Lists; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec; +import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.bean.entity.param.reassign.ExecuteReassignParam; +import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignExecutionStrategy; +import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignTask; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant; +import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.ClusterBalanceInterval; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.detail.ClusterBalancePlanDetail; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobConfigPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.ClusterBalanceHistorySubVO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.ClusterBalanceHistoryVO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter.ClusterBalanceConverter; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter.ClusterBalanceReassignConverter; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobConfigService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceReassignService; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService; +import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService; +import com.xiaojukeji.know.streaming.km.core.service.config.ConfigUtils; +import com.xiaojukeji.know.streaming.km.core.service.partition.OpPartitionService; +import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignService; +import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignStrategyService; +import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; +import com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance.ClusterBalanceJobDao; +import com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance.ClusterBalanceReassignDao; +import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BrokerBalanceState; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.interceptor.TransactionAspectSupport; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@EnterpriseLoadReBalance +public class ClusterBalanceJobServiceImpl implements ClusterBalanceJobService { + private static final ILog logger = LogFactory.getLog(ClusterBalanceJobServiceImpl.class); + + @Value("${es.client.address}") + private String esAddress; + + @Autowired + private ClusterBalanceJobDao clusterBalanceJobDao; + + @Autowired + private ClusterBalanceReassignDao clusterBalanceReassignDao; + + @Autowired + private ClusterBalanceReassignService clusterBalanceReassignService; + + @Autowired + private ClusterBalanceJobConfigService clusterBalanceJobConfigService; + + @Autowired + private BrokerSpecService brokerSpecService; + + @Autowired + private BrokerService brokerService; + + @Autowired + private ClusterPhyService clusterPhyService; + + @Autowired + private TopicService topicService; + + @Autowired + private ConfigUtils configUtils; + + @Autowired + private ReassignService reassignService; + + @Autowired + private ReassignStrategyService reassignStrategyService; + + @Autowired + private OpPartitionService opPartitionService; + + @Override + public Result deleteByJobId(Long jobId, String operator) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "jobId不允许为空"); + } + + try { + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, String.format("jobId:[%d] not exist", jobId)); + } + + if (JobStatusEnum.canNotDeleteJob(jobPO.getStatus())) { + // 状态错误，禁止执行 + return this.buildActionForbidden(jobId, jobPO.getStatus()); + } + + clusterBalanceJobDao.deleteById(jobId); + + LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); + lambdaQueryWrapper.eq(ClusterBalanceReassignPO::getJobId, jobId); + clusterBalanceReassignDao.delete(lambdaQueryWrapper); + return Result.buildSuc(); + } catch (Exception e) { + logger.error("method=delete||jobId={}||errMsg=exception", jobId, e); + + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage()); + } + } + + @Override + public Result createClusterBalanceJob(ClusterBalanceJobPO clusterBalanceJobPO, String operator) { + if (ValidateUtils.isNull(clusterBalanceJobPO)){ + return Result.buildFrom(ResultStatus.NOT_EXIST); + } + try { + clusterBalanceJobDao.addClusterBalanceJob(clusterBalanceJobPO); + }catch (Exception e){ + logger.error("method=createClusterBalanceJob||clusterBalanceJobPO:{}||errMsg=exception", clusterBalanceJobPO, e); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + + } + return Result.buildSuc(); + } + + @Override + public Result modifyClusterBalanceJob(ClusterBalanceJobPO clusterBalanceJobPO, String operator) { + + ClusterBalanceJobPO oldJobPo = clusterBalanceJobDao.selectById(clusterBalanceJobPO.getId()); + if (oldJobPo == null){ + return Result.buildFrom(ResultStatus.NOT_EXIST); + } + try { + int count = clusterBalanceJobDao.updateById(clusterBalanceJobPO); + if (count < 1){ + logger.error("method=modifyClusterBalanceJob||clusterBalanceJobPO:{}||errMsg=modify clusterBalanceJob failed", clusterBalanceJobPO); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + }catch (Exception e){ + logger.error("method=modifyClusterBalanceJob||clusterBalanceJobPO:{}||errMsg=exception", clusterBalanceJobPO, e); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + return Result.buildSuc(); + } + + @Override + public Result getClusterBalanceJobById(Long id) { + return Result.buildSuc(clusterBalanceJobDao.selectById(id)); + } + + @Override + public ClusterBalanceJobPO getLastOneByClusterId(Long clusterPhyId) { + ClusterBalanceJobPO clusterBalanceJobPO = new ClusterBalanceJobPO(); + clusterBalanceJobPO.setClusterId(clusterPhyId); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.setEntity(clusterBalanceJobPO); + queryWrapper.orderByDesc("id"); + List clusterBalanceJobPOS = clusterBalanceJobDao.selectList(queryWrapper); + if (clusterBalanceJobPOS.isEmpty()){ + return null; + } + return clusterBalanceJobPOS.get(0); + } + + @Override + public Map getBalanceInterval(Long clusterPhyId) { + Result configPOResult = clusterBalanceJobConfigService.getByClusterId(clusterPhyId); + if (!configPOResult.hasData()){ + return new HashMap(); + } + List clusterBalanceIntervals = ConvertUtil.str2ObjArrayByJson(configPOResult.getData().getBalanceIntervalJson(), ClusterBalanceInterval.class); + + return clusterBalanceIntervals.stream().collect(Collectors.toMap(ClusterBalanceInterval::getType,ClusterBalanceInterval::getIntervalPercent)); + } + + @Override + public PaginationResult page(Long clusterPhyId, PaginationBaseDTO dto) { + List historyVOS = new ArrayList<>(); + + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(ClusterBalanceJobPO::getClusterId, clusterPhyId); + List status = Lists.newArrayList(JobStatusEnum.SUCCESS.getStatus(), JobStatusEnum.CANCELED.getStatus(), JobStatusEnum.FAILED.getStatus()); + queryWrapper.in(ClusterBalanceJobPO::getStatus, status); + queryWrapper.orderByDesc(ClusterBalanceJobPO::getStartTime); + + IPage page = clusterBalanceJobDao.selectPage(new Page<>(dto.getPageNo(), dto.getPageSize()), queryWrapper); + page.setTotal(clusterBalanceJobDao.selectCount(queryWrapper)); + + for (ClusterBalanceJobPO clusterBalanceJobPO : page.getRecords()){ + ClusterBalanceHistoryVO clusterBalanceHistoryVO = new ClusterBalanceHistoryVO(); + clusterBalanceHistoryVO.setBegin(clusterBalanceJobPO.getStartTime()); + clusterBalanceHistoryVO.setEnd(clusterBalanceJobPO.getFinishedTime()); + clusterBalanceHistoryVO.setJobId(clusterBalanceJobPO.getId()); + + List detailVOS = ConvertUtil.str2ObjArrayByJson(clusterBalanceJobPO.getBrokerBalanceDetail(), ClusterBalancePlanDetail.class); + Map subMap = new HashMap<>(); + ClusterBalanceHistorySubVO diskSubVO = new ClusterBalanceHistorySubVO(); + diskSubVO.setSuccessNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getDiskStatus() != null && clusterBalancePlanDetail.getDiskStatus() == 0).count()); + diskSubVO.setFailedNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getDiskStatus() != null && clusterBalancePlanDetail.getDiskStatus() != 0).count()); + subMap.put(Resource.DISK.resource(), diskSubVO); + + ClusterBalanceHistorySubVO cupSubVO = new ClusterBalanceHistorySubVO(); + cupSubVO.setSuccessNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getCpuStatus() != null && clusterBalancePlanDetail.getCpuStatus() == 0).count()); + cupSubVO.setFailedNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getCpuStatus() != null && clusterBalancePlanDetail.getCpuStatus() != 0).count()); + subMap.put(Resource.CPU.resource(), cupSubVO); + + ClusterBalanceHistorySubVO bytesInSubVO = new ClusterBalanceHistorySubVO(); + bytesInSubVO.setSuccessNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getByteInStatus() != null && clusterBalancePlanDetail.getByteInStatus() == 0).count()); + bytesInSubVO.setFailedNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getByteInStatus() != null && clusterBalancePlanDetail.getByteInStatus() != 0).count()); + subMap.put(Resource.NW_IN.resource(), bytesInSubVO); + + ClusterBalanceHistorySubVO bytesOutSubVO = new ClusterBalanceHistorySubVO(); + bytesOutSubVO.setSuccessNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getByteOutStatus() != null && clusterBalancePlanDetail.getByteOutStatus() == 0).count()); + bytesOutSubVO.setFailedNu(detailVOS.stream().filter(clusterBalancePlanDetail -> clusterBalancePlanDetail.getByteOutStatus() != null && clusterBalancePlanDetail.getByteOutStatus() != 0).count()); + subMap.put(Resource.NW_OUT.resource(), bytesOutSubVO); + + clusterBalanceHistoryVO.setSub(subMap); + + historyVOS.add(clusterBalanceHistoryVO); + } + + return PaginationResult.buildSuc(historyVOS, page); + } + + @Override + public Long getOneRunningJob(Long clusterPhyId) { + LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); + lambdaQueryWrapper.eq(ClusterBalanceJobPO::getClusterId, clusterPhyId); + lambdaQueryWrapper.eq(ClusterBalanceJobPO::getStatus, JobStatusEnum.RUNNING.getStatus()); + + List poList = clusterBalanceJobDao.selectList(lambdaQueryWrapper); + if (!ValidateUtils.isEmptyList(poList)) { + // 默认获取第一个 + return poList.get(0).getId(); + } + + // 获取子任务中待执行的任务，避免主任务和子任务状态不一致 + LambdaQueryWrapper subLambdaQueryWrapper = new LambdaQueryWrapper<>(); + subLambdaQueryWrapper.eq(ClusterBalanceReassignPO::getClusterId, clusterPhyId); + subLambdaQueryWrapper.eq(ClusterBalanceReassignPO::getStatus, JobStatusEnum.RUNNING.getStatus()); + List subPOList = clusterBalanceReassignDao.selectList(subLambdaQueryWrapper); + if (ValidateUtils.isEmptyList(subPOList)) { + return null; + } + return subPOList.get(0).getJobId(); + } + + @Override + @Transactional + public Result verifyClusterBalanceAndUpdateStatue(Long jobId) { + ClusterBalanceJobPO clusterBalanceJobPO = clusterBalanceJobDao.selectById(jobId); + if (clusterBalanceJobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, String.format("jobId:[%d] not exist", jobId)); + } + + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterBalanceJobPO.getClusterId()); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + Result rv = reassignService.changReassignmentThrottles( + new ExecuteReassignParam(clusterBalanceJobPO.getClusterId(), clusterBalanceJobPO.getReassignmentJson(), clusterBalanceJobPO.getThrottleUnitB()) + ); + if (rv.failed()) { + logger.error("method=verifyClusterBalanceAndUpdateStatue||jobId={}||result={}||msg=change throttle failed", jobId, rv); + return rv; + } + + //获取规格信息 + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterBalanceJobPO.getClusterId()); + //获取broker信息 + Map brokerMap = brokerService.listAllBrokersFromDB(clusterBalanceJobPO.getClusterId()).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity())); + + //更新平衡任务状态信息 + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhy.getId(), configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + Map brokerBalanceStateMap = ExecutionRebalance + .getBrokerResourcesBalanceState(ClusterBalanceConverter.convert2BalanceParameter(clusterBalanceJobPO, brokerMap, brokerSpecMap, clusterPhy, esAddress, topicNames)); + List oldDetails = ConvertUtil.str2ObjArrayByJson(clusterBalanceJobPO.getBrokerBalanceDetail(), ClusterBalancePlanDetail.class); + List newDetails = ClusterBalanceConverter.convert2ClusterBalancePlanDetail(oldDetails, brokerBalanceStateMap); + clusterBalanceJobPO.setBrokerBalanceDetail(ConvertUtil.obj2Json(newDetails)); + Result modifyResult = this.modifyClusterBalanceJob(clusterBalanceJobPO, Constant.SYSTEM); + if (modifyResult.failed()){ + logger.error("method=verifyClusterBalanceAndUpdateStatue||jobId:{}||errMsg={}", jobId, modifyResult); + return modifyResult; + } + + //更新迁移任务状态信息 + Result result = clusterBalanceReassignService.verifyAndUpdateStatue(clusterBalanceJobPO); + if (!result.hasData()){ + return Result.buildFromIgnoreData(result); + } + + rv = clusterBalanceReassignService.preferredReplicaElection(jobId); + if (rv.failed()){ + logger.error("method=verifyClusterBalanceAndUpdateStatue||jobId={}||result={}||msg=preferred replica election failed", jobId, rv); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return rv; + } + + return Result.buildSuc(); + } + + @Override + public Result generateReassignmentJson(Long clusterId, + Integer parallelNum, + Integer executionStrategy, + Integer jsonVersion, + List reassignPOList){ + Result> result = reassignStrategyService.generateReassignmentTask( + new ReassignExecutionStrategy( + clusterId, + parallelNum, + executionStrategy, + ClusterBalanceReassignConverter.convert2ReplaceReassignSubList(reassignPOList) + ) + ); + + if (result.failed() || result.getData().isEmpty()){ + return Result.buildFromIgnoreData(result); + } + + Map reassign = new HashMap<>(); + reassign.put(KafkaConstant.PARTITIONS, result.getData()); + reassign.put(KafkaConstant.VERSION, jsonVersion); + String generateReassignmentJson = ConvertUtil.obj2Json(reassign); + + // 检查生成的迁移Json是否合法 + Result rv = reassignService.parseExecuteAssignmentArgs(clusterId, generateReassignmentJson); + if (rv.failed()) { + return Result.buildFromIgnoreData(rv); + } + return Result.buildSuc(generateReassignmentJson); + } + + @Override + @Transactional + public Result generateReassignmentForStrategy(Long clusterPhyId, Long jobId) { + ClusterBalanceJobPO job = clusterBalanceJobDao.selectById(jobId); + if (job == null){ + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, + MsgConstant.getReassignJobBizStr(jobId, clusterPhyId)); + } + if (!JobStatusEnum.isRunning(job.getStatus()) || job.getParallelNum() < 1){ + return Result.buildSuc(); + } + + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + List reassignPOS = clusterBalanceReassignService.getBalanceReassignsByJobId(jobId); + + //2.4以内版本因为无法动态增加副本迁移，所以需要等一部分任务完成之后再根据并行度重新获下一部分迁移任务 + Double version = new Double(clusterPhy.getKafkaVersion().substring(0,3)); + if (version < 2.4 && reassignPOS.stream() + .filter(reassignPO -> reassignPO.getStatus()==JobStatusEnum.RUNNING.getStatus()).count() > 0){ + return Result.buildSuc(); + } + + //过滤已完成子任务 + reassignPOS = reassignPOS.stream() + .filter(reassignPO -> reassignPO.getStatus()==JobStatusEnum.RUNNING.getStatus() + ||reassignPO.getStatus()==JobStatusEnum.WAITING.getStatus()) + .collect(Collectors.toList()); + if (reassignPOS.isEmpty()){ + return Result.buildSuc(); + } + + Map reassign = JSON.parseObject(job.getBalanceIntervalJson()); + Result r = this.generateReassignmentJson(job.getClusterId(), + job.getParallelNum(), + job.getExecutionStrategy(), + reassign.get(KafkaConstant.VERSION)==null?1:(Integer)reassign.get(KafkaConstant.VERSION) + 1, + reassignPOS); + if (!r.hasData()){ + return Result.buildFromIgnoreData(r); + } + try { + //更新任务json + job.setReassignmentJson(r.getData()); + job.setUpdateTime(new Date()); + clusterBalanceJobDao.updateById(job); + + //更新任务状态 + modifyReassignStatus(r.getData(), reassignPOS); + }catch (Exception e){ + logger.error("method=generateReassignmentForStrategy||jobId={}||errMsg=exception", jobId, e); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + + return reassignService.executePartitionReassignments( + new ExecuteReassignParam(job.getClusterId(), r.getData(), job.getThrottleUnitB())); + + } + + private Result buildActionForbidden(Long jobId, Integer jobStatus) { + return Result.buildFromRSAndMsg( + ResultStatus.OPERATION_FORBIDDEN, + String.format("jobId:[%d] 当前 status:[%s], 不允许被执行", jobId, JobStatusEnum.valueOfStatus(jobStatus)) + ); + } + + private void modifyReassignStatus(String reassignmentJson, List reassignPOS){ + Map reassign = JSON.parseObject(reassignmentJson); + List reassignTasks = (List)reassign.get(KafkaConstant.PARTITIONS); + if (reassignTasks == null || reassignTasks.isEmpty()){ + return; + } + + // 更新子任务状态 + reassignTasks.forEach(reassignTask -> { + for (ClusterBalanceReassignPO reassignPO: reassignPOS) { + if (reassignPO.getStatus().equals(JobStatusEnum.WAITING.getStatus()) + && reassignTask.getTopic().equals(reassignPO.getTopicName()) + && reassignTask.getPartition() == reassignPO.getPartitionId()) { + ClusterBalanceReassignPO newReassignPO = new ClusterBalanceReassignPO(); + newReassignPO.setId(reassignPO.getId()); + newReassignPO.setStatus(JobStatusEnum.RUNNING.getStatus()); + newReassignPO.setUpdateTime(new Date()); + clusterBalanceReassignDao.updateById(newReassignPO); + break; + } + } + }); + } + +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceReassignServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceReassignServiceImpl.java new file mode 100644 index 00000000..0bd3834a --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceReassignServiceImpl.java @@ -0,0 +1,514 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.impl; + +import com.alibaba.fastjson.JSON; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.didiglobal.logi.security.common.dto.oplog.OplogDTO; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.ClusterBalanceReassignDetail; +import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics; +import com.xiaojukeji.know.streaming.km.common.bean.entity.param.reassign.ExecuteReassignParam; +import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition; +import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignResult; +import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReassignSubJobExtendData; +import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignTask; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant; +import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter.ClusterBalanceReassignConverter; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum; +import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum; +import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum; +import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceReassignService; +import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService; +import com.xiaojukeji.know.streaming.km.core.service.partition.OpPartitionService; +import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService; +import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignService; +import com.xiaojukeji.know.streaming.km.core.service.replica.ReplicaMetricService; +import com.xiaojukeji.know.streaming.km.core.service.version.metrics.ReplicaMetricVersionItems; +import com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance.ClusterBalanceJobDao; +import com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance.ClusterBalanceReassignDao; +import org.apache.kafka.common.TopicPartition; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.interceptor.TransactionAspectSupport; + +import java.util.*; + +@Service +@EnterpriseLoadReBalance +public class ClusterBalanceReassignServiceImpl implements ClusterBalanceReassignService { + private static final ILog logger = LogFactory.getLog(ClusterBalanceReassignServiceImpl.class); + + @Autowired + private ClusterBalanceReassignDao clusterBalanceReassignDao; + + @Autowired + private ClusterBalanceJobDao clusterBalanceJobDao; + + @Autowired + private ReassignService reassignService; + + @Autowired + private ReplicaMetricService replicationMetricService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private OpLogWrapService opLogWrapService; + + @Autowired + private OpPartitionService opPartitionService; + + @Override + public Result addBalanceReassign(ClusterBalanceReassignPO clusterBalanceReassignPO) { + if (clusterBalanceReassignPO == null) { + return Result.buildFrom(ResultStatus.NOT_EXIST); + } + try { + int count = clusterBalanceReassignDao.insert(clusterBalanceReassignPO); + if (count < 1) { + logger.error("create cluster balance reassign detail failed! clusterBalanceReassignPO:{}", clusterBalanceReassignPO); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + } catch (Exception e) { + logger.error("create cluster balance reassign detail failed! clusterBalanceReassignPO:{}", clusterBalanceReassignPO, e); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + + } + return Result.buildSuc(); + } + + @Override + public Result addBatchBalanceReassign(List reassignPOList) { + try { + int count = clusterBalanceReassignDao.addBatch(reassignPOList); + if (count < 1) { + logger.error("method=addBatchBalanceReassign||reassignPOList:{}||msg=create cluster balance reassign detail failed! ", reassignPOList); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + } catch (Exception e) { + logger.error("method=addBatchBalanceReassign||reassignPOList:{}||msg=create cluster balance reassign detail failed! ", reassignPOList, e); + return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED); + } + return Result.buildSuc(); + } + + @Override + public Result delete(Long jobId, String operator) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull()); + } + + try { + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + + LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); + lambdaQueryWrapper.eq(ClusterBalanceReassignPO::getJobId, jobId); + clusterBalanceReassignDao.delete(lambdaQueryWrapper); + return Result.buildSuc(); + } catch (Exception e) { + logger.error("method=delete||jobId={}||errMsg=exception", jobId, e); + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage()); + } + } + + @Override + public Result execute(Long jobId) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull()); + } + + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + + if (!JobStatusEnum.canExecuteJob(jobPO.getStatus())) { + // 状态错误，禁止执行 + return this.buildActionForbidden(jobId, jobPO.getStatus()); + } + + // 修改DB状态 + this.setJobInRunning(jobPO); + + // 执行任务 + Result rv = reassignService.executePartitionReassignments(new ExecuteReassignParam(jobPO.getClusterId(), jobPO.getReassignmentJson(), jobPO.getThrottleUnitB())); + if (rv.failed()) { + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return rv; + } + + return Result.buildSuc(); + } + + @Override + public Result cancel(Long jobId) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull()); + } + + try { + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + + if (!JobStatusEnum.canCancelJob(jobPO.getStatus())) { + // 状态错误，禁止执行 + return this.buildActionForbidden(jobId, jobPO.getStatus()); + } + + this.setJobCanceled(jobPO); + + return Result.buildSuc(); + } catch (Exception e) { + logger.error("method=cancel||jobId={}||errMsg=exception", jobId, e); + + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage()); + } + } + + @Override + @Transactional + public Result verifyAndUpdateStatue(ClusterBalanceJobPO jobPO) { + if (jobPO == null) { + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, "job not exist"); + } + + // 检查迁移的结果 + Result reassignResult = reassignService.verifyPartitionReassignments( + new ExecuteReassignParam(jobPO.getClusterId(), jobPO.getReassignmentJson(), jobPO.getThrottleUnitB()) + ); + + if (reassignResult.failed()) { + return Result.buildFromIgnoreData(reassignResult); + } + + // 更新任务状态 + return this.checkAndSetSuccessIfFinished(jobPO, reassignResult.getData()); + } + + @Override + public Result modifyThrottle(Long jobId, Long throttleUnitB, String operator) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull()); + } + + try { + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + + if (JobStatusEnum.isFinished(jobPO.getStatus())) { + // 状态错误，禁止执行 + return this.buildActionForbidden(jobId, jobPO.getStatus()); + } + + // 修改限流值 + jobPO.setThrottleUnitB(throttleUnitB); + clusterBalanceJobDao.updateById(jobPO); + + // 记录操作 + opLogWrapService.saveOplogAndIgnoreException(new OplogDTO( + operator, + OperationEnum.EDIT.getDesc(), + ModuleEnum.JOB_CLUSTER_BALANCE.getDesc(), + MsgConstant.getReassignJobBizStr( + jobId, + jobPO.getClusterId() + ), + String.format("新的限流值:[%d]", throttleUnitB) + )); + + return Result.buildSuc(); + } catch (Exception e) { + logger.error("method=modifyThrottle||jobId={}||throttleUnitB={}||errMsg=exception", jobId, throttleUnitB, e); + } + + return Result.buildSuc(); + } + + @Override + public Result getAndUpdateSubJobExtendData(Long jobId) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull()); + } + + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + + List reassigns = this.getBalanceReassignsByJobId(jobId); + for (ClusterBalanceReassignPO reassignPO: reassigns) { + Result extendDataResult = this.getReassignSubJobExtendData(reassignPO); + if (extendDataResult.failed()) { + continue; + } + + reassignPO.setExtendData(ConvertUtil.obj2Json(extendDataResult.getData())); + clusterBalanceReassignDao.updateById(reassignPO); + } + + return Result.buildSuc(); + } + + @Override + public List getBalanceReassignsByJobId(Long jobId) { + if (jobId == null) { + return new ArrayList<>(); + } + + LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); + lambdaQueryWrapper.eq(ClusterBalanceReassignPO::getJobId, jobId); + + return clusterBalanceReassignDao.selectList(lambdaQueryWrapper); + } + + @Override + public Result getJobDetailsGroupByTopic(Long jobId) { + if (jobId == null) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull()); + } + + // 获取任务 + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + + // 获取子任务 + List subJobPOList = this.getBalanceReassignsByJobId(jobId); + + // 数据组装并放回 + return Result.buildSuc(ClusterBalanceReassignConverter.convert2ClusterBalanceReassignDetail(jobPO, subJobPOList)); + } + + @Override + public Result preferredReplicaElection(Long jobId) { + // 获取任务 + ClusterBalanceJobPO jobPO = clusterBalanceJobDao.selectById(jobId); + if (jobPO == null) { + // 任务不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId)); + } + if (!JobStatusEnum.isFinished(jobPO.getStatus())){ + return Result.buildSuc(); + } + + // 获取子任务 + List subJobPOList = this.getBalanceReassignsByJobId(jobId); + List topicPartitions = new ArrayList<>(); + subJobPOList.stream().forEach(reassignPO -> { + Integer targetLeader = CommonUtils.string2IntList(reassignPO.getReassignBrokerIds()).get(0); + Integer originalLeader = CommonUtils.string2IntList(reassignPO.getOriginalBrokerIds()).get(0); + //替换过leader的添加到优先副本选举任务列表 + if (!originalLeader.equals(targetLeader)){ + topicPartitions.add(new TopicPartition(reassignPO.getTopicName(), reassignPO.getPartitionId())); + } + }); + + if (!topicPartitions.isEmpty()){ + return opPartitionService.preferredReplicaElection(jobPO.getClusterId(), topicPartitions); + } + + return Result.buildSuc(); + } + + private Result buildActionForbidden(Long jobId, Integer jobStatus) { + return Result.buildFromRSAndMsg( + ResultStatus.OPERATION_FORBIDDEN, + String.format("jobId:[%d] 当前 status:[%s], 不允许被执行", jobId, JobStatusEnum.valueOfStatus(jobStatus)) + ); + } + + private Result setJobInRunning(ClusterBalanceJobPO jobPO) { + Map reassign = JSON.parseObject(jobPO.getReassignmentJson()); + if (reassign.isEmpty()){ + return Result.buildSuc(); + } + + List reassignTasks = JSON.parseArray(JSON.toJSONString(reassign.get(KafkaConstant.PARTITIONS)), ReassignTask.class); + if (reassignTasks == null || reassignTasks.isEmpty()){ + return Result.buildSuc(); + } + + long now = System.currentTimeMillis(); + // 更新子任务状态 + List reassignPOS = this.getBalanceReassignsByJobId(jobPO.getId()); + reassignTasks.forEach(reassignTask -> { + for (ClusterBalanceReassignPO reassignPO: reassignPOS) { + if (reassignTask.getTopic().equals(reassignPO.getTopicName()) + && reassignTask.getPartition() == reassignPO.getPartitionId()) { + ClusterBalanceReassignPO newReassignPO = new ClusterBalanceReassignPO(); + newReassignPO.setId(reassignPO.getId()); + newReassignPO.setStatus(JobStatusEnum.RUNNING.getStatus()); + newReassignPO.setStartTime(new Date(now)); + newReassignPO.setUpdateTime(new Date(now)); + clusterBalanceReassignDao.updateById(newReassignPO); + break; + } + } + }); + + // 更新父任务状态 + ClusterBalanceJobPO newJobPO = new ClusterBalanceJobPO(); + newJobPO.setId(jobPO.getId()); + newJobPO.setStatus(JobStatusEnum.RUNNING.getStatus()); + newJobPO.setStartTime(new Date(now)); + newJobPO.setUpdateTime(new Date(now)); + clusterBalanceJobDao.updateById(newJobPO); + + return Result.buildSuc(); + } + + private Result setJobCanceled(ClusterBalanceJobPO jobPO) { + // 更新子任务状态 + List reassignPOS = this.getBalanceReassignsByJobId(jobPO.getId()); + for (ClusterBalanceReassignPO reassignPO: reassignPOS) { + ClusterBalanceReassignPO newReassignPO = new ClusterBalanceReassignPO(); + newReassignPO.setId(reassignPO.getId()); + newReassignPO.setStatus(JobStatusEnum.CANCELED.getStatus()); + clusterBalanceReassignDao.updateById(newReassignPO); + } + + // 更新父任务状态 + ClusterBalanceJobPO newJobPO = new ClusterBalanceJobPO(); + newJobPO.setId(jobPO.getId()); + newJobPO.setStatus(JobStatusEnum.CANCELED.getStatus()); + clusterBalanceJobDao.updateById(newJobPO); + + return Result.buildSuc(); + } + + private Result checkAndSetSuccessIfFinished(ClusterBalanceJobPO jobPO, ReassignResult reassignmentResult) { + long now = System.currentTimeMillis(); + List reassignPOS = this.getBalanceReassignsByJobId(jobPO.getId()); + boolean existNotFinished = false; + for (ClusterBalanceReassignPO balanceReassignPO: reassignPOS) { + if (!reassignmentResult.checkPartitionFinished(balanceReassignPO.getTopicName(), balanceReassignPO.getPartitionId())) { + existNotFinished = true; + continue; + } + + // 更新状态 + ClusterBalanceReassignPO newReassignPO = new ClusterBalanceReassignPO(); + newReassignPO.setId(balanceReassignPO.getId()); + newReassignPO.setStatus(JobStatusEnum.SUCCESS.getStatus()); + newReassignPO.setFinishedTime(new Date(now)); + clusterBalanceReassignDao.updateById(newReassignPO); + } + + // 更新任务状态 + if (!existNotFinished && !reassignmentResult.isPartsOngoing()) { + ClusterBalanceJobPO newBalanceJobPO = new ClusterBalanceJobPO(); + newBalanceJobPO.setId(jobPO.getId()); + newBalanceJobPO.setStatus(JobStatusEnum.SUCCESS.getStatus()); + newBalanceJobPO.setFinishedTime(new Date(now)); + clusterBalanceJobDao.updateById(newBalanceJobPO); + } + + return Result.buildSuc(reassignmentResult.isPartsOngoing()); + } + + private Result getReassignSubJobExtendData(ClusterBalanceReassignPO subJobPO) { + Partition partition = partitionService.getPartitionByTopicAndPartitionId( + subJobPO.getClusterId(), + subJobPO.getTopicName(), + subJobPO.getPartitionId() + ); + + if (partition == null) { + // 分区不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getPartitionNotExist(subJobPO.getClusterId(), + subJobPO.getTopicName(), + subJobPO.getPartitionId()) + ); + } + + // 获取leader副本 + Float leaderLogSize = this.getReplicaLogSize(subJobPO.getClusterId(), + partition.getLeaderBrokerId(), + subJobPO.getTopicName(), + subJobPO.getPartitionId() + ); + + // 获取新增的副本 + Set newReplicas = new HashSet<>(CommonUtils.string2IntList(subJobPO.getReassignBrokerIds())); + newReplicas.removeAll(CommonUtils.string2IntList(subJobPO.getOriginalBrokerIds())); + + Long finishedLogSizeUnitB = 0L; + for (Integer brokerId: newReplicas) { + Float replicaLogSize = this.getReplicaLogSize(subJobPO.getClusterId(), + brokerId, + subJobPO.getTopicName(), + subJobPO.getPartitionId() + ); + if (replicaLogSize == null) { + continue; + } + + finishedLogSizeUnitB += replicaLogSize.longValue(); + } + + ReassignSubJobExtendData extendData = ConvertUtil.str2ObjByJson(subJobPO.getExtendData(), ReassignSubJobExtendData.class); + extendData.setFinishedReassignLogSizeUnitB(finishedLogSizeUnitB); + if (leaderLogSize != null) { + extendData.setNeedReassignLogSizeUnitB(leaderLogSize.longValue() * newReplicas.size()); + } + //迁移任务已完成时，若分区指标未更新，已完成logSize等于需要迁移logSize + if (JobStatusEnum.isFinished(subJobPO.getStatus()) && finishedLogSizeUnitB.equals(0L)){ + extendData.setFinishedReassignLogSizeUnitB(extendData.getNeedReassignLogSizeUnitB()); + } + + if (extendData.getNeedReassignLogSizeUnitB().equals(0L) || JobStatusEnum.isFinished(subJobPO.getStatus())) { + extendData.setRemainTimeUnitMs(0L); + } else if (extendData.getFinishedReassignLogSizeUnitB().equals(0L)) { + // 未知 + extendData.setRemainTimeUnitMs(null); + } else { + Long usedTime = System.currentTimeMillis() - subJobPO.getStartTime().getTime(); + // (需迁移LogSize / 已迁移LogSize) = (总时间 / 已进行时间) + extendData.setRemainTimeUnitMs(extendData.getNeedReassignLogSizeUnitB() * usedTime / extendData.getFinishedReassignLogSizeUnitB()); + } + + return Result.buildSuc(extendData); + } + + private Float getReplicaLogSize(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId) { + Result replicaMetricsResult = replicationMetricService.collectReplicaMetricsFromKafka( + clusterPhyId, + topicName, + partitionId, + brokerId, + ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE + ); + + if (!replicaMetricsResult.hasData()) { + return null; + } + + return replicaMetricsResult.getData().getMetric(ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE); + } +} \ No newline at end of file diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceServiceImpl.java new file mode 100644 index 00000000..47ba9caa --- /dev/null +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/enterprise/rebalance/service/impl/ClusterBalanceServiceImpl.java @@ -0,0 +1,481 @@ +package com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.impl; + +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceIntervalDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceOverviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalancePreviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceStrategyDTO; +import com.xiaojukeji.know.streaming.km.common.bean.dto.job.JobDTO; +import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; +import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec; +import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.ClusterBalanceItemState; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.job.content.JobClusterBalanceContent; +import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobConfigPO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.*; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.converter.ClusterBalanceConverter; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.enums.ClusterBalanceStateEnum; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobHandleEnum; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum; +import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum; +import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; +import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService; +import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobConfigService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceService; +import com.xiaojukeji.know.streaming.km.core.service.config.ConfigUtils; +import com.xiaojukeji.know.streaming.km.core.service.job.JobService; +import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; +import com.xiaojukeji.know.streaming.km.core.service.version.metrics.BrokerMetricVersionItems; +import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BrokerBalanceState; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult; +import org.apache.logging.log4j.core.util.CronExpression; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.text.ParseException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@EnterpriseLoadReBalance +public class ClusterBalanceServiceImpl implements ClusterBalanceService { + private static final ILog logger = LogFactory.getLog(ClusterBalanceServiceImpl.class); + + @Value("${es.client.address}") + private String esAddress; + + @Autowired + private JobService jobService; + + @Autowired + private ClusterBalanceJobService clusterBalanceJobService; + + @Autowired + private ClusterPhyService clusterPhyService; + + @Autowired + private BrokerService brokerService; + + @Autowired + private BrokerSpecService brokerSpecService; + + @Autowired + private ClusterBalanceJobConfigService clusterBalanceJobConfigService; + + @Autowired + private BrokerMetricService brokerMetricService; + + @Autowired + private TopicService topicService; + + @Autowired + private ConfigUtils configUtils; + + @Override + public Result state(Long clusterPhyId) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + Result configPOResult = clusterBalanceJobConfigService.getByClusterId(clusterPhyId); + if(!configPOResult.hasData()){ + return Result.buildFromIgnoreData(configPOResult); + } + + Map brokerMap = brokerService.listAllBrokersFromDB(clusterPhy.getId()).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity())); + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhy.getId()); + + ClusterBalanceStateVO clusterBalanceStateVO = new ClusterBalanceStateVO(); + try { + CronExpression cronExpression = new CronExpression(configPOResult.getData().getTaskCron()); + //是否到满足周期时间 + clusterBalanceStateVO.setNext(cronExpression.getTimeAfter(new Date())); + } catch (ParseException e) { + logger.error("method=state||clusterId:{}||errMsg=exception", clusterPhyId, e); + } + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + + clusterBalanceStateVO.setEnable(configPOResult.getData().getStatus() == 1); + Map resourceDoubleMap; + Map brokerBalanceStateMap; + try { + resourceDoubleMap = ExecutionRebalance.getClusterAvgResourcesState(ClusterBalanceConverter.convert2BalanceParameter(configPOResult.getData(), brokerMap, brokerSpecMap, clusterPhy, esAddress, topicNames)); + brokerBalanceStateMap = ExecutionRebalance + .getBrokerResourcesBalanceState(ClusterBalanceConverter.convert2BalanceParameter(configPOResult.getData(), brokerMap, brokerSpecMap, clusterPhy, esAddress, topicNames)); + }catch (Exception e){ + logger.error("method=state||clusterPhyId={}||errMsg=exception", clusterPhyId, e); + return Result.buildFailure(e.getMessage()); + } + //集群状态信息 + ArrayList balanceStates = new ArrayList(brokerBalanceStateMap.values()); + clusterBalanceStateVO.setStatus(ClusterBalanceStateEnum.BALANCE.getState()); + balanceStates.forEach(brokerBalanceState ->{ + if ((brokerBalanceState.getDiskBalanceState() != null && !brokerBalanceState.getDiskBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())) + || (brokerBalanceState.getCpuBalanceState() != null && !brokerBalanceState.getCpuBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())) + || (brokerBalanceState.getBytesOutBalanceState() != null && !brokerBalanceState.getBytesOutBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState())) + || (brokerBalanceState.getBytesInBalanceState() != null && !brokerBalanceState.getBytesInBalanceState().equals(ClusterBalanceStateEnum.BALANCE.getState()))){ + clusterBalanceStateVO.setStatus(ClusterBalanceStateEnum.UNBALANCED.getState()); + } + }); + clusterBalanceStateVO.setSub(getStateSubVOMap(resourceDoubleMap, balanceStates, clusterPhyId)); + return Result.buildSuc(clusterBalanceStateVO); + } + + @Override + public Result config(Long clusterPhyId) { + Result configPOResult = clusterBalanceJobConfigService.getByClusterId(clusterPhyId); + if (!configPOResult.hasData()){ + return Result.buildFromIgnoreData(configPOResult); + } + return Result.buildSuc(ClusterBalanceConverter.convert2ClusterBalanceJobConfigVO(configPOResult.getData())); + } + + @Override + public PaginationResult overview(Long clusterPhyId, ClusterBalanceOverviewDTO dto) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return PaginationResult.buildFailure(ResultStatus.CLUSTER_NOT_EXIST, dto); + } + Result configPOResult = clusterBalanceJobConfigService.getByClusterId(clusterPhyId); + if(configPOResult.failed()){ + return PaginationResult.buildFailure(configPOResult, dto); + } + + //获取规格信息 + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhyId); + List clusterBalanceOverviewVOS = new ArrayList<>(); + List brokerList = brokerService.listAllBrokersFromDB(clusterPhyId); + Map brokerMap = brokerList.stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity())); + Map brokerBalanceStateMap = new HashMap<>(); + if (configPOResult.hasData()) { + try { + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + brokerBalanceStateMap = ExecutionRebalance + .getBrokerResourcesBalanceState(ClusterBalanceConverter.convert2BalanceParameter(configPOResult.getData(), brokerMap, brokerSpecMap, clusterPhy, esAddress, topicNames)); + } catch (Exception e) { + logger.error("method=overview||clusterBalanceOverviewDTO={}||errMsg=exception", dto, e); + return PaginationResult.buildFailure(e.getMessage(), dto); + } + } + + // 获取指标 + Result> metricsResult = brokerMetricService.getLatestMetricsFromES( + clusterPhyId, + brokerList.stream().filter(elem1 -> elem1.alive()).map(elem2 -> elem2.getBrokerId()).collect(Collectors.toList()) + ); + if (metricsResult.failed()){ + return PaginationResult.buildFailure(metricsResult, dto); + } + Map brokerMetricsMap = new HashMap<>(); + if (metricsResult.hasData()){ + brokerMetricsMap = metricsResult.getData().stream().collect(Collectors.toMap(BrokerMetrics::getBrokerId, Function.identity())); + } + + for(Map.Entry entry : brokerMap.entrySet()){ + Broker broker = entry.getValue(); + if (broker == null){ + continue; + } + ClusterBalanceOverviewVO clusterBalanceOverviewVO = new ClusterBalanceOverviewVO(); + clusterBalanceOverviewVO.setBrokerId(entry.getKey()); + clusterBalanceOverviewVO.setHost(broker.getHost()); + clusterBalanceOverviewVO.setRack(broker.getRack()); + BrokerMetrics brokerMetrics = brokerMetricsMap.get(entry.getKey()); + if (brokerMetrics != null){ + clusterBalanceOverviewVO.setLeader(brokerMetrics.getMetric( BrokerMetricVersionItems.BROKER_METRIC_LEADERS)!=null + ?brokerMetrics.getMetric( BrokerMetricVersionItems.BROKER_METRIC_LEADERS).intValue():null); + clusterBalanceOverviewVO.setReplicas(brokerMetrics.getMetric( BrokerMetricVersionItems.BROKER_METRIC_PARTITIONS)!=null + ?brokerMetrics.getMetric( BrokerMetricVersionItems.BROKER_METRIC_PARTITIONS).intValue():null); + } + clusterBalanceOverviewVO.setSub(ClusterBalanceConverter.convert2MapClusterBalanceOverviewSubVO(brokerSpecMap.get(entry.getKey()), brokerBalanceStateMap.get(entry.getKey()))); + clusterBalanceOverviewVOS.add(clusterBalanceOverviewVO); + } + + //过滤status + if (dto.getStateParam()!= null && dto.getStateParam().size()>0){ + clusterBalanceOverviewVOS = filterState(dto.getStateParam(), clusterBalanceOverviewVOS); + } + + clusterBalanceOverviewVOS = PaginationUtil.pageByFuzzyFilter(ConvertUtil.list2List(clusterBalanceOverviewVOS, ClusterBalanceOverviewVO.class), dto.getSearchKeywords(), Arrays.asList("host")); + + return PaginationResult.buildSuc(clusterBalanceOverviewVOS, clusterBalanceOverviewVOS.size(), dto.getPageNo(), dto.getPageSize()); + } + + @Override + public Result getItemState(Long clusterPhyId) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + ClusterBalanceItemState clusterBalanceState = new ClusterBalanceItemState(); + Result configPOResult = clusterBalanceJobConfigService.getByClusterId(clusterPhyId); + if(!configPOResult.hasData()){ + clusterBalanceState.setConfigureBalance(Boolean.FALSE); + clusterBalanceState.setEnable(Boolean.FALSE); + return Result.buildSuc(clusterBalanceState); + } + + Map brokerMap = brokerService.listAllBrokersFromDB(clusterPhy.getId()).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity())); + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhy.getId()); + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + + clusterBalanceState.setConfigureBalance(Boolean.TRUE); + clusterBalanceState.setEnable(configPOResult.getData().getStatus() == 1); + Map brokerBalanceStateMap; + try { + brokerBalanceStateMap = ExecutionRebalance + .getBrokerResourcesBalanceState(ClusterBalanceConverter.convert2BalanceParameter(configPOResult.getData(), brokerMap, brokerSpecMap, clusterPhy, esAddress, topicNames)); + }catch (Exception e){ + logger.error("method=state||clusterPhyId={}||errMsg=exception", clusterPhyId, e); + return Result.buildFailure(e.getMessage()); + } + + Map itemStateMap = new HashMap<>(); + //集群状态信息 + ArrayList balanceStates = new ArrayList(brokerBalanceStateMap.values()); + List intervalDTOS = ConvertUtil.str2ObjArrayByJson(configPOResult.getData().getBalanceIntervalJson(), ClusterBalanceIntervalDTO.class); + intervalDTOS.forEach(intervalDTO->{ + if (Resource.CPU.resource().equals(intervalDTO.getType())){ + itemStateMap.put(Resource.CPU.resource(), balanceStates.stream() + .filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getCpuBalanceState())).count()==brokerMap.size()); + }else if (Resource.NW_IN.resource().equals(intervalDTO.getType())){ + itemStateMap.put(Resource.NW_IN.resource(), balanceStates.stream() + .filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getBytesInBalanceState())).count()==brokerMap.size()); + }else if (Resource.NW_OUT.resource().equals(intervalDTO.getType())){ + itemStateMap.put(Resource.NW_OUT.resource(), balanceStates.stream() + .filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getBytesOutBalanceState())).count()==brokerMap.size()); + }else if (Resource.DISK.resource().equals(intervalDTO.getType())){ + itemStateMap.put(Resource.DISK.resource(), balanceStates.stream() + .filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getDiskBalanceState())).count()==brokerMap.size()); + + } + }); + clusterBalanceState.setItemState(itemStateMap); + + return Result.buildSuc(clusterBalanceState); + } + + private List filterState(Map stateParam, List oldVos){ + if (stateParam.isEmpty()){ + return oldVos; + } + List overviewVOS = new ArrayList<>(); + for(ClusterBalanceOverviewVO oldVo : oldVos){ + Boolean check = true; + for(Map.Entry paramEntry : stateParam.entrySet()){ + ClusterBalanceOverviewSubVO subVO = oldVo.getSub().get(paramEntry.getKey()); + if (subVO == null){ + check = false; + continue; + } + if (subVO.getStatus()==null || !subVO.getStatus().equals(paramEntry.getValue())){ + check = false; + } + } + if (check){ + overviewVOS.add(oldVo); + } + } + + return overviewVOS; + } + + @Override + public PaginationResult history(Long clusterPhyId, PaginationBaseDTO dto) { + return clusterBalanceJobService.page(clusterPhyId, dto); + } + + @Override + public Result plan(Long clusterPhyId, Long jobId) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + Result jobPOResult = clusterBalanceJobService.getClusterBalanceJobById(jobId); + if (jobPOResult.failed()){ + return Result.buildFrom(ResultStatus.NOT_EXIST); + } + + List allBrokers= brokerService.listAllBrokersFromDB(clusterPhyId); + + ClusterBalancePlanVO planVO = new ClusterBalancePlanVO(); + ClusterBalanceJobPO jobPO = jobPOResult.getData(); + planVO.setMoveSize(jobPO.getTotalReassignSize()); + planVO.setBrokers(ClusterBalanceConverter.convert2HostList(allBrokers, jobPO.getBrokers())); + planVO.setBlackTopics(CommonUtils.string2StrList(jobPO.getTopicBlackList())); + planVO.setReplicas(jobPO.getTotalReassignReplicaNum()); + planVO.setType(jobPO.getType()); + planVO.setTopics(CommonUtils.string2StrList(jobPO.getMoveInTopicList())); + planVO.setDetail(ConvertUtil.str2ObjArrayByJson(jobPO.getBrokerBalanceDetail(), ClusterBalancePlanDetailVO.class)); + planVO.setReassignmentJson(jobPO.getReassignmentJson()); + planVO.setClusterBalanceIntervalList(ConvertUtil.str2ObjArrayByJson(jobPO.getBalanceIntervalJson(), ClusterBalanceIntervalVO.class)); + return Result.buildSuc(planVO); + } + + @Override + public Result preview(Long clusterPhyId, ClusterBalancePreviewDTO clusterBalancePreviewDTO) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + List allBrokers = brokerService.listAllBrokersFromDB(clusterPhy.getId()); + Map brokerSpecMap = brokerSpecService.getBrokerSpecMap(clusterPhy.getId()); + for(Broker broker:allBrokers){ + if (brokerSpecMap.get(broker.getBrokerId()) == null){ + return Result.buildFromRSAndMsg(ResultStatus.BROKER_SPEC_NOT_EXIST,String.format("Broker规格信息不存在:brokerId:%s", broker.getBrokerId())); + } + } + + if (clusterBalancePreviewDTO.getBrokers() == null || clusterBalancePreviewDTO.getBrokers().isEmpty()){ + clusterBalancePreviewDTO.setBrokers( + allBrokers.stream().map(Broker::getBrokerId).collect(Collectors.toList())); + } + + //获取任务计划 + Map brokerMap = allBrokers.stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity())); + List topicNames = topicService.listRecentUpdateTopicNamesFromDB(clusterPhyId, configUtils.getClusterBalanceIgnoredTopicsTimeSecond()); + BalanceParameter balanceParameter = ClusterBalanceConverter.convert2BalanceParameter(clusterBalancePreviewDTO, brokerMap, brokerSpecMap, clusterPhy, esAddress, topicNames); + ExecutionRebalance executionRebalance = new ExecutionRebalance(); + try { + OptimizerResult optimizerResult = executionRebalance.optimizations(balanceParameter); + if (optimizerResult == null) { + return Result.buildFrom(ResultStatus.KAFKA_OPERATE_FAILED); + } + + //生成平衡job + return Result.buildSuc(ClusterBalanceConverter.convert2ClusterBalancePlanVO(clusterBalancePreviewDTO, optimizerResult, allBrokers)); + } catch (Exception e){ + logger.error("method=preview||clusterBalancePreviewDTO:{}||errMsg=exception", clusterBalancePreviewDTO, e); + return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage()); + } + } + + @Override + public Result schedule(Long clusterPhyId, Long jobId) { + Result rbr= clusterBalanceJobService.getClusterBalanceJobById(jobId); + if (!rbr.hasData()){ + return Result.buildFromIgnoreData(rbr); + } + + return preview(clusterPhyId, ClusterBalanceConverter.convert2ClusterBalancePreviewDTO(rbr.getData())); + } + + @Override + public Result strategy(Long clusterPhyId, ClusterBalanceStrategyDTO dto, String operator) { + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId); + if (clusterPhy == null){ + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + //如果不是周期任务，那么就直接往 jobService 中添加一个任务 + if(!dto.isScheduleJob()){ + JobDTO jobDTO = new JobDTO(); + jobDTO.setPlanTime(new Date()); + jobDTO.setJobStatus(JobStatusEnum.WAITING.getStatus()); + jobDTO.setCreator(operator); + jobDTO.setJobType(JobHandleEnum.CLUSTER_BALANCE.getType()); + jobDTO.setTarget(JobHandleEnum.CLUSTER_BALANCE.getMessage()); + jobDTO.setJobData(ConvertUtil.obj2Json(dto)); + return jobService.addTask(clusterPhyId, jobDTO, operator); + }else { + return clusterBalanceJobConfigService.replaceClusterBalanceJobConfigByClusterId(ClusterBalanceConverter.convert2ClusterBalanceJobConfigPO(dto, operator)); + } + } + + @Override + public Result createScheduleJob(Long clusterPhyId, long triggerTimeUnitMs){ + //获取到 clusterPhyId 对应的周期任务策略 + Result configPOResult = clusterBalanceJobConfigService.getByClusterId(clusterPhyId); + if (!configPOResult.hasData() || configPOResult.getData().getStatus().equals(Constant.DOWN)){ + return Result.buildSuc(); + } + + try { + CronExpression cronExpression = new CronExpression(configPOResult.getData().getTaskCron()); + //是否到满足周期时间 + if (!cronExpression.isSatisfiedBy(new Date(triggerTimeUnitMs))){ + return Result.buildSuc(); + } + } catch (ParseException e) { + logger.error("method=createScheduleJob||clusterId:{}||errMsg=exception", clusterPhyId, e); + + e.printStackTrace(); + } + + //满足周期时间新增job任务 + JobDTO jobDTO = new JobDTO(); + jobDTO.setPlanTime(new Date()); + jobDTO.setJobStatus(JobStatusEnum.WAITING.getStatus()); + jobDTO.setCreator(Constant.SYSTEM); + jobDTO.setJobType(JobTypeEnum.CLUSTER_BALANCE.getType()); + jobDTO.setTarget(JobHandleEnum.CLUSTER_BALANCE.getMessage()); + JobClusterBalanceContent content = ClusterBalanceConverter.convert2JobClusterBalanceContent(configPOResult.getData()); + jobDTO.setJobData(ConvertUtil.obj2Json(content)); + return jobService.addTask(clusterPhyId, jobDTO, Constant.SYSTEM); + } + + private Map getStateSubVOMap(Map resourceDoubleMap, ArrayList balanceStates, Long clusterId){ + Map subVOMap = new HashMap<>(); + Map balanceInterval = clusterBalanceJobService.getBalanceInterval(clusterId); + for (Map.Entry entry : resourceDoubleMap.entrySet()){ + Resource resource = entry.getKey(); + if (Resource.CPU.resource().equals(resource.resource())){ + ClusterBalanceStateSubVO cpuSubVo = new ClusterBalanceStateSubVO(); + cpuSubVo.setAvg(entry.getValue()); + cpuSubVo.setInterval(balanceInterval.get(Resource.CPU.resource())); + cpuSubVo.setBetweenNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getCpuBalanceState())).count()); + cpuSubVo.setBigNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.ABOVE_BALANCE.getState().equals(status.getCpuBalanceState())).count()); + cpuSubVo.setSmallNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BELOW_BALANCE.getState().equals(status.getCpuBalanceState())).count()); + subVOMap.put(Resource.CPU.resource(), cpuSubVo); + }else if (Resource.NW_IN.resource().equals(resource.resource())){ + ClusterBalanceStateSubVO cpuSubVo = new ClusterBalanceStateSubVO(); + cpuSubVo.setAvg(entry.getValue()); + cpuSubVo.setInterval(balanceInterval.get(Resource.NW_IN.resource())); + cpuSubVo.setBetweenNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getBytesInBalanceState())).count()); + cpuSubVo.setBigNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.ABOVE_BALANCE.getState().equals(status.getBytesInBalanceState())).count()); + cpuSubVo.setSmallNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BELOW_BALANCE.getState().equals(status.getBytesInBalanceState())).count()); + subVOMap.put(Resource.NW_IN.resource(), cpuSubVo); + }else if (Resource.NW_OUT.resource().equals(resource.resource())){ + ClusterBalanceStateSubVO cpuSubVo = new ClusterBalanceStateSubVO(); + cpuSubVo.setAvg(entry.getValue()); + cpuSubVo.setInterval(balanceInterval.get(Resource.NW_OUT.resource())); + cpuSubVo.setBetweenNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getBytesOutBalanceState())).count()); + cpuSubVo.setBigNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.ABOVE_BALANCE.getState().equals(status.getBytesOutBalanceState())).count()); + cpuSubVo.setSmallNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BELOW_BALANCE.getState().equals(status.getBytesOutBalanceState())).count()); + subVOMap.put(Resource.NW_OUT.resource(), cpuSubVo); + }else if (Resource.DISK.resource().equals(resource.resource())){ + ClusterBalanceStateSubVO cpuSubVo = new ClusterBalanceStateSubVO(); + cpuSubVo.setAvg(entry.getValue()); + cpuSubVo.setInterval(balanceInterval.get(Resource.DISK.resource())); + cpuSubVo.setBetweenNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BALANCE.getState().equals(status.getDiskBalanceState())).count()); + cpuSubVo.setBigNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.ABOVE_BALANCE.getState().equals(status.getDiskBalanceState())).count()); + cpuSubVo.setSmallNu(balanceStates.stream().filter(status->ClusterBalanceStateEnum.BELOW_BALANCE.getState().equals(status.getDiskBalanceState())).count()); + subVOMap.put(Resource.DISK.resource(), cpuSubVo); + } + } + return subVOMap; + } +} diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterMetricServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterMetricServiceImpl.java index bdd652aa..c3a0c3e5 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterMetricServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterMetricServiceImpl.java @@ -5,10 +5,12 @@ import com.didiglobal.logi.log.LogFactory; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.collect.Table; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO; import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsClusterPhyDTO; import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.entity.ClusterBalanceItemState; import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController; import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics; import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics; @@ -38,6 +40,7 @@ import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService; import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterMetricService; import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceService; import com.xiaojukeji.know.streaming.km.core.service.group.GroupService; import com.xiaojukeji.know.streaming.km.core.service.health.state.HealthStateService; import com.xiaojukeji.know.streaming.km.core.service.job.JobService; @@ -50,6 +53,7 @@ import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache; import com.xiaojukeji.know.streaming.km.persistence.es.dao.ClusterMetricESDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; import org.apache.kafka.common.resource.ResourceType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; @@ -69,12 +73,14 @@ import java.util.stream.Collectors; import static com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics.initWithMetrics; import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*; +import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.ClusterMetricVersionItems.*; import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.TopicMetricVersionItems.*; /** * @author didi */ @Service("clusterMetricService") +@EnterpriseLoadReBalance(all = false) public class ClusterMetricServiceImpl extends BaseMetricService implements ClusterMetricService { private static final ILog LOGGER = LogFactory.getLog(ClusterMetricServiceImpl.class); @@ -113,6 +119,9 @@ public class ClusterMetricServiceImpl extends BaseMetricService implements Clust public static final String CLUSTER_METHOD_GET_JOBS_SUCCESS = "getJobsSuccess"; public static final String CLUSTER_METHOD_GET_JOBS_FAILED = "getJobsFailed"; + @EnterpriseLoadReBalance(all = false) + public static final String CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO = "getClusterLoadReBalanceInfo"; + @Autowired private HealthStateService healthStateService; @@ -152,6 +161,9 @@ public class ClusterMetricServiceImpl extends BaseMetricService implements Clust @Autowired private JobService jobService; + @Autowired + private ClusterBalanceService clusterBalanceService; + @Autowired private ClusterPhyService clusterPhyService; @@ -179,6 +191,7 @@ public class ClusterMetricServiceImpl extends BaseMetricService implements Clust } @Override + @EnterpriseLoadReBalance(all = false) protected void initRegisterVCHandler(){ registerVCHandler( CLUSTER_METHOD_DO_NOTHING, this::doNothing); registerVCHandler( CLUSTER_METHOD_GET_TOPIC_SIZE, this::getTopicSize); @@ -218,6 +231,8 @@ public class ClusterMetricServiceImpl extends BaseMetricService implements Clust registerVCHandler( CLUSTER_METHOD_GET_JOBS_WAITING, this::getJobsWaiting); registerVCHandler( CLUSTER_METHOD_GET_JOBS_SUCCESS, this::getJobsSuccess); registerVCHandler( CLUSTER_METHOD_GET_JOBS_FAILED, this::getJobsFailed); + + registerVCHandler( CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO, this::getClusterLoadReBalanceInfo); } @Override @@ -676,6 +691,26 @@ public class ClusterMetricServiceImpl extends BaseMetricService implements Clust return Result.buildSuc(initWithMetrics(clusterId, metric, count)); } + @EnterpriseLoadReBalance + private Result getClusterLoadReBalanceInfo(VersionItemParam metricParam) { + ClusterMetricParam param = (ClusterMetricParam)metricParam; + + Result stateResult = clusterBalanceService.getItemState(param.getClusterId()); + if (stateResult.failed()) { + return Result.buildFromIgnoreData(stateResult); + } + + ClusterBalanceItemState state = stateResult.getData(); + + ClusterMetrics metric = ClusterMetrics.initWithMetrics(param.getClusterId(), CLUSTER_METRIC_LOAD_RE_BALANCE_ENABLE, state.getEnable()? Constant.YES: Constant.NO); + metric.putMetric(CLUSTER_METRIC_LOAD_RE_BALANCE_CPU, state.getResItemState(Resource.CPU).floatValue()); + metric.putMetric(CLUSTER_METRIC_LOAD_RE_BALANCE_NW_IN, state.getResItemState(Resource.NW_IN).floatValue()); + metric.putMetric(CLUSTER_METRIC_LOAD_RE_BALANCE_NW_OUT, state.getResItemState(Resource.NW_OUT).floatValue()); + metric.putMetric(CLUSTER_METRIC_LOAD_RE_BALANCE_DISK, state.getResItemState(Resource.DISK).floatValue()); + + return Result.buildSuc(metric); + } + /** * 从某一个 controller 的 JMX 中获取指标再聚合得到集群的指标 * @param metricParam diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/config/ConfigUtils.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/config/ConfigUtils.java index 58c26b69..079d069b 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/config/ConfigUtils.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/config/ConfigUtils.java @@ -1,6 +1,7 @@ package com.xiaojukeji.know.streaming.km.core.service.config; import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -13,4 +14,7 @@ import org.springframework.stereotype.Service; public class ConfigUtils { private ConfigUtils() { } + + @Value("${cluster-balance.ignored-topics.time-second:300}") + private Integer clusterBalanceIgnoredTopicsTimeSecond; } diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/metrics/ClusterMetricVersionItems.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/metrics/ClusterMetricVersionItems.java index ea81da3a..a7618def 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/metrics/ClusterMetricVersionItems.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/metrics/ClusterMetricVersionItems.java @@ -1,5 +1,6 @@ package com.xiaojukeji.know.streaming.km.core.service.version.metrics; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem; import com.xiaojukeji.know.streaming.km.common.constant.Constant; import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum; @@ -18,6 +19,7 @@ import static com.xiaojukeji.know.streaming.km.core.service.cluster.impl.Cluster * @author didi */ @Component +@EnterpriseLoadReBalance(all = false) public class ClusterMetricVersionItems extends BaseMetricVersionMetric { /** * 整体的健康指标 @@ -103,6 +105,13 @@ public class ClusterMetricVersionItems extends BaseMetricVersionMetric { public static final String CLUSTER_METRIC_JOB_SUCCESS = "JobsSuccess"; public static final String CLUSTER_METRIC_JOB_FAILED = "JobsFailed"; + @EnterpriseLoadReBalance + public static final String CLUSTER_METRIC_LOAD_RE_BALANCE_ENABLE = "LoadReBalanceEnable"; + public static final String CLUSTER_METRIC_LOAD_RE_BALANCE_CPU = "LoadReBalanceCpu"; + public static final String CLUSTER_METRIC_LOAD_RE_BALANCE_NW_IN = "LoadReBalanceNwIn"; + public static final String CLUSTER_METRIC_LOAD_RE_BALANCE_NW_OUT = "LoadReBalanceNwOut"; + public static final String CLUSTER_METRIC_LOAD_RE_BALANCE_DISK = "LoadReBalanceDisk"; + public static final String CLUSTER_METRIC_COLLECT_COST_TIME = Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME; public ClusterMetricVersionItems(){} @@ -113,6 +122,7 @@ public class ClusterMetricVersionItems extends BaseMetricVersionMetric { } @Override + @EnterpriseLoadReBalance(all = false) public List init(){ List itemList = new ArrayList<>(); @@ -397,6 +407,27 @@ public class ClusterMetricVersionItems extends BaseMetricVersionMetric { .name(CLUSTER_METRIC_JOB_FAILED).unit("个").desc("集群failed任务总数").category(CATEGORY_JOB) .extend( buildMethodExtend( CLUSTER_METHOD_GET_JOBS_FAILED ))); + // 集群维度-均衡相关 + itemList.add( buildAllVersionsItem() + .name(CLUSTER_METRIC_LOAD_RE_BALANCE_ENABLE).unit("是/否").desc("是否开启均衡, 1：是；0：否").category(CATEGORY_CLUSTER) + .extend( buildMethodExtend( CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO ))); + + itemList.add( buildAllVersionsItem() + .name(CLUSTER_METRIC_LOAD_RE_BALANCE_CPU).unit("是/否").desc("CPU是否均衡, 1：是；0：否").category(CATEGORY_CLUSTER) + .extend( buildMethodExtend( CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO ))); + + itemList.add( buildAllVersionsItem() + .name(CLUSTER_METRIC_LOAD_RE_BALANCE_NW_IN).unit("是/否").desc("BytesIn是否均衡, 1：是；0：否").category(CATEGORY_CLUSTER) + .extend( buildMethodExtend( CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO ))); + + itemList.add( buildAllVersionsItem() + .name(CLUSTER_METRIC_LOAD_RE_BALANCE_NW_OUT).unit("是/否").desc("BytesOut是否均衡, 1：是；0：否").category(CATEGORY_CLUSTER) + .extend( buildMethodExtend( CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO ))); + + itemList.add( buildAllVersionsItem() + .name(CLUSTER_METRIC_LOAD_RE_BALANCE_DISK).unit("是/否").desc("Disk是否均衡, 1：是；0：否").category(CATEGORY_CLUSTER) + .extend( buildMethodExtend( CLUSTER_METHOD_GET_CLUSTER_LOAD_RE_BALANCE_INFO ))); + itemList.add(buildAllVersionsItem() .name(CLUSTER_METRIC_COLLECT_COST_TIME).unit("秒").desc("采集Cluster指标的耗时").category(CATEGORY_PERFORMANCE) .extendMethod(CLUSTER_METHOD_DO_NOTHING)); diff --git a/km-enterprise/km-testing/pom.xml b/km-enterprise/km-testing/pom.xml new file mode 100644 index 00000000..ebafbbc9 --- /dev/null +++ b/km-enterprise/km-testing/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + com.xiaojukeji.kafka + km-testing + ${km.revision} + jar + + + km + com.xiaojukeji.kafka + ${km.revision} + ../../pom.xml + + + + + com.xiaojukeji.kafka + km-common + ${project.parent.version} + + + + com.xiaojukeji.kafka + km-core + ${project.parent.version} + + + + + diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/KmTestingApplication.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/KmTestingApplication.java new file mode 100644 index 00000000..77069987 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/KmTestingApplication.java @@ -0,0 +1,12 @@ +package com.xiaojukeji.know.streaming.km.testing; + +import org.springframework.boot.SpringApplication; + + +public class KmTestingApplication { + + public static void main(String[] args) { + SpringApplication.run(KmTestingApplication.class, args); + } + +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/KafkaClientTestManager.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/KafkaClientTestManager.java new file mode 100644 index 00000000..9d95d555 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/KafkaClientTestManager.java @@ -0,0 +1,30 @@ +package com.xiaojukeji.know.streaming.km.testing.biz; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaConsumerDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaProducerDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestConsumerVO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestProducerVO; + + +import java.util.List; + +@EnterpriseTesting +public interface KafkaClientTestManager { + /** + * 生产测试 + * @param dto 生产测试参数 + * @param operator 操作人 + * @return + */ + Result> produceTest(KafkaProducerDTO dto, String operator); + + /** + * 消费测试 + * @param dto 消费测试参数 + * @param operator 操作人 + * @return + */ + Result consumeTest(KafkaConsumerDTO dto, String operator); +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/impl/KafkaClientTestManagerImpl.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/impl/KafkaClientTestManagerImpl.java new file mode 100644 index 00000000..3c4f3e9f --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/impl/KafkaClientTestManagerImpl.java @@ -0,0 +1,596 @@ +package com.xiaojukeji.know.streaming.km.testing.biz.impl; + +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.didiglobal.logi.security.common.dto.oplog.OplogDTO; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.dto.partition.PartitionOffsetDTO; +import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.bean.entity.record.RecordHeaderKS; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus; +import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic; +import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO; +import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant; +import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant; +import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum; +import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum; +import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException; +import com.xiaojukeji.know.streaming.km.common.exception.NotExistException; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; +import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService; +import com.xiaojukeji.know.streaming.km.core.service.group.GroupService; +import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService; +import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService; +import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; +import com.xiaojukeji.know.streaming.km.testing.biz.KafkaClientTestManager; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaConsumerDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaConsumerFilterDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaConsumerStartFromDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaProducerDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestConsumerVO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestPartitionConsumedVO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestProducerVO; +import com.xiaojukeji.know.streaming.km.testing.common.enums.KafkaConsumerFilterEnum; +import com.xiaojukeji.know.streaming.km.testing.common.enums.KafkaConsumerStartFromEnum; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +@Component +@EnterpriseTesting +public class KafkaClientTestManagerImpl implements KafkaClientTestManager { + private static final ILog log = LogFactory.getLog(KafkaClientTestManagerImpl.class); + + @Autowired + private TopicService topicService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private GroupService groupService; + + @Autowired + private ClusterPhyService clusterPhyService; + + @Autowired + private OpLogWrapService opLogWrapService; + + @Override + public Result consumeTest(KafkaConsumerDTO dto, String operator) { + if (ValidateUtils.anyNull(dto, operator)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + // 获取集群信息 + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterId()); + if (clusterPhy == null) { + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getClusterId())); + } + + // 检查start 和 filter两个参数是否合法 + Result rv = this.checkStartFromAndFilterLegal(dto.getStartFrom(), dto.getFilter()); + if (rv.failed()) { + // 参数错误 + return Result.buildFromIgnoreData(rv); + } + + KafkaConsumer kafkaConsumer = null; + try { + // 获取消费测试开始消费的offset信息 + Result> consumeStartOffsetResult = this.getConsumeStartOffset(dto.getClusterId(), dto.getTopicName(), dto.getStartFrom()); + if (consumeStartOffsetResult.failed()) { + // 获取offset失败 + return Result.buildFromIgnoreData(consumeStartOffsetResult); + } + + //获取topic的BeginOffset + Result> partitionBeginOffsetMapResult = partitionService.getPartitionOffsetFromKafka(dto.getClusterId(), dto.getTopicName(), OffsetSpec.earliest(), null); + if (partitionBeginOffsetMapResult.failed()) { + return Result.buildFromIgnoreData(partitionBeginOffsetMapResult); + } + + //计算最终的开始offset + consumeStartOffsetResult.getData().forEach(elem -> { + long offset = Math.max(partitionBeginOffsetMapResult.getData().get(new TopicPartition(dto.getTopicName(), elem.getPartitionId())), elem.getOffset()); + elem.setOffset(offset); + }); + + // 获取Topic的EndOffset + Result> partitionEndOffsetMapResult = partitionService.getPartitionOffsetFromKafka(dto.getClusterId(), dto.getTopicName(), OffsetSpec.latest(), null); + if (partitionEndOffsetMapResult.failed()) { + return Result.buildFromIgnoreData(partitionEndOffsetMapResult); + } + + // 创建Consumer客户端 + Properties properties = this.buildProperties(clusterPhy, dto.getClientProperties(), true); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, dto.getMaxRecords()); + kafkaConsumer = new KafkaConsumer<>(properties); + + // 消费数据 + List recordList = this.fetchData( + kafkaConsumer, + dto.getMaxDurationUnitMs(), + dto.getMaxRecords(), + dto.getTopicName(), + consumeStartOffsetResult.getData(), + partitionEndOffsetMapResult.getData() + ); + + // 进行数据组装 + Result voResult = Result.buildSuc(this.convert2TestConsumerVO( + dto.getTopicName(), + recordList, + partitionEndOffsetMapResult.getData(), + consumeStartOffsetResult.getData(), + dto.getFilter()) + ); + + // 记录用户操作 + if (voResult.successful() && dto.getRecordOperate()) { + opLogWrapService.saveOplogAndIgnoreException(new OplogDTO( + operator, + OperationEnum.SEARCH.getDesc(), + ModuleEnum.KAFKA_TOPIC_DATA.getDesc(), + dto.getTopicName(), + MsgConstant.getTopicBizStr(dto.getClusterId(), dto.getTopicName()) + )); + } + + return voResult; + } catch (NotExistException nee) { + log.error("method=consumeTest||param={}||operator={}||errMsg=res not exist.", dto, operator, nee); + + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage()); + } catch (Exception e) { + log.error("method=consumeTest||param={}||operator={}||errMsg=exception.", dto, operator, e); + + return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage()); + } finally { + if (kafkaConsumer != null) { + kafkaConsumer.close(); + } + } + } + + @Override + public Result> produceTest(@Validated KafkaProducerDTO dto, String operator) { + if (ValidateUtils.anyNull(dto, operator)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + // 获取集群信息 + ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterId()); + if (clusterPhy == null) { + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getClusterId())); + } + + // 内部Topic不允许生产 + if (KafkaConstant.KAFKA_INTERNAL_TOPICS.contains(dto.getTopicName())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "内部Topic不允许进行生产测试"); + } + + KafkaProducer kafkaProducer = null; + try { + // 获取Topic信息并检查分区信息是否合法 + Topic topic = topicService.getTopic(clusterPhy.getId(), dto.getTopicName()); + if (topic == null) { + log.error("method=produceTest||param={}||operator={}||errMsg=res not exist.", dto, operator); + + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(clusterPhy.getId(), dto.getTopicName())); + } + + if (!ValidateUtils.isEmptyList(dto.getPartitionIdList()) && dto.getPartitionIdList().stream().anyMatch(elem -> !topic.getPartitionMap().containsKey(elem))) { + // 分区不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getPartitionNotExist(dto.getClusterId(), dto.getTopicName())); + } + + // 创建生产客户端 + kafkaProducer = new KafkaProducer<>(this.buildProperties(clusterPhy, dto.getClientProperties(), false)); + + // 进行数据生产 + Result> listResult = this.sendData(kafkaProducer, dto); + + // 如果成功，并且需要进行记录，则记录操作 + if (listResult.successful() && dto.getRecordOperate()) { + opLogWrapService.saveOplogAndIgnoreException(new OplogDTO( + operator, + OperationEnum.ADD.getDesc(), + ModuleEnum.KAFKA_TOPIC_DATA.getDesc(), + dto.getTopicName(), + MsgConstant.getTopicBizStr(dto.getClusterId(), dto.getTopicName()) + )); + } + + // 返回操作结果 + return listResult; + } catch (Exception e) { + log.error("method=produceTest||param={}||operator={}||errMsg=exception!", dto, operator, e); + + return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage()); + } finally { + if (kafkaProducer != null) { + kafkaProducer.close(); + } + } + } + + /**************************************************** private method ****************************************************/ + + private Result> sendData(KafkaProducer kafkaProducer, KafkaProducerDTO dto) throws InterruptedException, ExecutionException { + List headers = new ArrayList<>(); + if (dto.getRecordHeader() != null) { + for (Map.Entry entry: dto.getRecordHeader().entrySet()) { + headers.add(new RecordHeader(entry.getKey().toString(), entry.getValue().toString().getBytes(StandardCharsets.UTF_8))); + } + } + + // 随机一个数，如果指定分区了，则从该随机数下标位置的分区开始生产 + int idx = new Random().nextInt(10000); + + long now = System.currentTimeMillis(); + List> futureList = new ArrayList<>(); + for (int i = 0; i < dto.getRecordCount(); ++i) { + Integer partitionId = null; + if (!ValidateUtils.isEmptyList(dto.getPartitionIdList())) { + partitionId = dto.getPartitionIdList().get(idx % dto.getPartitionIdList().size()); + idx += 1; + } + if (headers.isEmpty()) { + futureList.add(kafkaProducer.send(new ProducerRecord(dto.getTopicName(), partitionId, dto.getRecordKey(), dto.getRecordValue()))); + } else { + futureList.add(kafkaProducer.send(new ProducerRecord(dto.getTopicName(), partitionId, dto.getRecordKey(), dto.getRecordValue(), headers))); + } + } + + kafkaProducer.flush(); + + List voList = new ArrayList<>(); + for (Future metadataFuture: futureList) { + RecordMetadata recordMetadata = metadataFuture.get(); + voList.add(new TestProducerVO( + recordMetadata.timestamp() - now, + recordMetadata.partition(), + recordMetadata.offset() == -1? null: recordMetadata.offset(), // 如果返回为-1，则设置为null + recordMetadata.timestamp()) + ); + } + + return Result.buildSuc(voList); + } + + private Properties buildProperties(ClusterPhy clusterPhy, Properties customProperties, boolean isConsume) { + Properties properties = ConvertUtil.str2ObjByJson(clusterPhy.getClientProperties(), Properties.class); + if (properties == null) { + properties = new Properties(); + } + properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, clusterPhy.getBootstrapServers()); + if (isConsume) { + // 反序列化 + properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + + // 默认禁止提交offset + properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + } else { + // 序列化 + properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + } + + if (customProperties != null) { + properties.putAll(customProperties); + } + return properties; + } + + private List fetchData(KafkaConsumer kafkaConsumer, + Long maxDurationUnitMs, + Integer maxRecords, + String topicName, + List dtoList, + Map endOffsetMap) { + long now = System.currentTimeMillis(); + + // 获取有数据的分区 + Map hasDataPartitionMap = dtoList + .stream() + .filter(elem -> { + Long endOffset = endOffsetMap.get(new TopicPartition(topicName, elem.getPartitionId())); + return endOffset != null && endOffset > elem.getOffset(); + }) + .collect(Collectors.toMap(PartitionOffsetDTO::getPartitionId, PartitionOffsetDTO::getOffset)); + + if (ValidateUtils.isEmptyMap(hasDataPartitionMap)) { + return new ArrayList<>(); + } + + // assign进行消费的分区 + kafkaConsumer.assign( + hasDataPartitionMap.keySet().stream().map(elem -> new TopicPartition(topicName, elem)).collect(Collectors.toList()) + ); + + // 设置消费的起始offset + for (Map.Entry entry: hasDataPartitionMap.entrySet()) { + kafkaConsumer.seek(new TopicPartition(topicName, entry.getKey()), entry.getValue()); + } + + List recordList = new ArrayList<>(); + while (System.currentTimeMillis() - now <= maxDurationUnitMs && recordList.size() < maxRecords) { + for (ConsumerRecord consumerRecord: kafkaConsumer.poll(Duration.ofSeconds(3))) { + recordList.add(consumerRecord); + if (recordList.size() >= maxRecords) { + break; + } + } + } + return recordList; + } + + private Result> getConsumeStartOffset(Long clusterPhyId, String topicName, KafkaConsumerStartFromDTO startFromDTO) throws NotExistException, AdminOperateException { + // 最新位置开始消费 + if (KafkaConsumerStartFromEnum.LATEST.getCode().equals(startFromDTO.getStartFromType())) { + Result> offsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, OffsetSpec.latest(), null); + if (offsetMapResult.failed()) { + return Result.buildFromIgnoreData(offsetMapResult); + } + + return Result.buildSuc(offsetMapResult.getData().entrySet() + .stream() + .map(entry-> new PartitionOffsetDTO(entry.getKey().partition(), entry.getValue())) + .collect(Collectors.toList()) + ); + } + + // 最旧位置开始消费 + if (KafkaConsumerStartFromEnum.EARLIEST.getCode().equals(startFromDTO.getStartFromType())) { + Result> offsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, OffsetSpec.earliest(), null); + if (offsetMapResult.failed()) { + return Result.buildFromIgnoreData(offsetMapResult); + } + + return Result.buildSuc(offsetMapResult.getData().entrySet() + .stream() + .map(entry-> new PartitionOffsetDTO(entry.getKey().partition(), entry.getValue())) + .collect(Collectors.toList()) + ); + } + + // 指定时间开始消费 + if (KafkaConsumerStartFromEnum.PRECISE_TIMESTAMP.getCode().equals(startFromDTO.getStartFromType())) { + Result> offsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, OffsetSpec.forTimestamp(startFromDTO.getTimestampUnitMs()), startFromDTO.getTimestampUnitMs()); + if (offsetMapResult.failed()) { + return Result.buildFromIgnoreData(offsetMapResult); + } + + return Result.buildSuc(offsetMapResult.getData().entrySet() + .stream() + .map(entry-> new PartitionOffsetDTO(entry.getKey().partition(), Math.max(entry.getValue(), 0L))) + .collect(Collectors.toList()) + ); + } + + // 指定位置开始消费 + if (KafkaConsumerStartFromEnum.PRECISE_OFFSET.getCode().equals(startFromDTO.getStartFromType())) { + return Result.buildSuc(startFromDTO.getOffsetList()); + } + + // 指定消费组进行消费 + if (KafkaConsumerStartFromEnum.CONSUMER_GROUP.getCode().equals(startFromDTO.getStartFromType())) { + Map offsetMap = groupService.getGroupOffsetFromKafka(clusterPhyId, startFromDTO.getConsumerGroup()); + return Result.buildSuc(offsetMap.entrySet() + .stream() + .filter(elem -> elem.getKey().topic().equals(topicName)) + .map(entry-> new PartitionOffsetDTO(entry.getKey().partition(), entry.getValue())) + .collect(Collectors.toList()) + ); + } + + // 近X条数据开始消费 + if (KafkaConsumerStartFromEnum.LATEST_MINUS_X_OFFSET.getCode().equals(startFromDTO.getStartFromType())) { + Result> offsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, OffsetSpec.latest(), null); + if (offsetMapResult.failed()) { + return Result.buildFromIgnoreData(offsetMapResult); + } + + return Result.buildSuc(offsetMapResult.getData().entrySet() + .stream() + .map(entry-> new PartitionOffsetDTO(entry.getKey().partition(), Math.max(0, entry.getValue() - startFromDTO.getLatestMinusX()))) + .collect(Collectors.toList()) + ); + } + + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "startFrom类型未知"); + } + + private Result checkStartFromAndFilterLegal(KafkaConsumerStartFromDTO startFrom, KafkaConsumerFilterDTO filter) { + // 指定时间开始消费 + if (KafkaConsumerStartFromEnum.PRECISE_TIMESTAMP.getCode().equals(startFrom.getStartFromType()) && ValidateUtils.isNullOrLessThanZero(startFrom.getTimestampUnitMs())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "指定时间消费必须设置时间参数"); + } + + // 指定位置开始消费 + if (KafkaConsumerStartFromEnum.PRECISE_OFFSET.getCode().equals(startFrom.getStartFromType()) && ValidateUtils.isEmptyList(startFrom.getOffsetList())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "指定offset消费必须设置offset参数"); + } + + // 指定消费组进行消费 + if (KafkaConsumerStartFromEnum.CONSUMER_GROUP.getCode().equals(startFrom.getStartFromType()) && ValidateUtils.isBlank(startFrom.getConsumerGroup())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "指定Group消费必须设置Group参数"); + } + + // 近X条数据开始消费 + if (KafkaConsumerStartFromEnum.LATEST_MINUS_X_OFFSET.getCode().equals(startFrom.getStartFromType()) && ValidateUtils.isNullOrLessThanZero(startFrom.getLatestMinusX())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "指定近X条开始消费必须设置latestMinusX参数"); + } + + // 包含过滤 + if (KafkaConsumerFilterEnum.CONTAINS.getCode().equals(filter.getFilterType()) + && ValidateUtils.isBlank(filter.getFilterCompareKey()) && ValidateUtils.isBlank(filter.getFilterCompareValue())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "包含的方式过滤，必须有过滤的key或value"); + } + + // 不包含过滤 + if (KafkaConsumerFilterEnum.NOT_CONTAINS.getCode().equals(filter.getFilterType()) + && ValidateUtils.isBlank(filter.getFilterCompareKey()) && ValidateUtils.isBlank(filter.getFilterCompareValue())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "不包含的方式过滤，必须有过滤的key或value"); + } + + // 等于过滤 + if (KafkaConsumerFilterEnum.EQUAL_SIZE.getCode().equals(filter.getFilterType()) && ValidateUtils.isNullOrLessThanZero(filter.getFilterCompareSizeUnitB())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "等于Size的方式过滤，必须有过滤size大小参数"); + } + + // size大于过滤 + if (KafkaConsumerFilterEnum.ABOVE_SIZE.getCode().equals(filter.getFilterType()) && ValidateUtils.isNullOrLessThanZero(filter.getFilterCompareSizeUnitB())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "大于Size的方式过滤，必须有过滤size大小参数"); + } + + // size小于过滤 + if (KafkaConsumerFilterEnum.UNDER_SIZE.getCode().equals(filter.getFilterType()) && ValidateUtils.isNullOrLessThanZero(filter.getFilterCompareSizeUnitB())) { + return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "小于Size的方式过滤，必须有过滤size大小参数"); + } + + return Result.buildSuc(); + } + + private TestConsumerVO convert2TestConsumerVO(String topicName, + List recordList, + Map partitionEndOffsetMap, + List consumedStartFromOffsetList, + KafkaConsumerFilterDTO filter) { + Map consumedVOMap = new HashMap<>(); + for (PartitionOffsetDTO partitionOffsetDTO: consumedStartFromOffsetList) { + TestPartitionConsumedVO vo = consumedVOMap.get(partitionOffsetDTO.getPartitionId()); + if (vo == null) { + vo = new TestPartitionConsumedVO(); + vo.setPartitionId(partitionOffsetDTO.getPartitionId()); + vo.setConsumedOffset(partitionOffsetDTO.getOffset()); + vo.setRecordSizeUnitB(0L); + vo.setRecordCount(0); + vo.setLogEndOffset(partitionEndOffsetMap.get(new TopicPartition(topicName, partitionOffsetDTO.getPartitionId()))); + consumedVOMap.put(partitionOffsetDTO.getPartitionId(), vo); + } + } + + TestConsumerVO vo = new TestConsumerVO(); + vo.setRecordList(new ArrayList<>()); + vo.setTotalRecordCount(0); + vo.setTotalRecordSizeUnitB(0L); + vo.setMaxRecordTimestampUnitMs(0L); + + for (ConsumerRecord record: recordList) { + // 统计消费信息 + TestPartitionConsumedVO consumedVO = consumedVOMap.get(record.partition()); + if (consumedVO == null) { + consumedVO = new TestPartitionConsumedVO(); + consumedVO.setRecordSizeUnitB(0L); + consumedVO.setRecordCount(0); + consumedVO.setPartitionId(record.partition()); + consumedVO.setLogEndOffset(partitionEndOffsetMap.get(new TopicPartition(topicName, record.partition()))); + + consumedVOMap.put(record.partition(), consumedVO); + } + + if (record.offset() > consumedVO.getConsumedOffset()) { + consumedVO.setConsumedOffset(record.offset() + 1); + } + + consumedVO.setRecordCount(consumedVO.getRecordCount() + 1); + consumedVO.setRecordSizeUnitB(consumedVO.getRecordSizeUnitB() + record.serializedKeySize() + record.serializedValueSize()); + + // 进行数据过滤 + if (this.checkMatchFilter(record, filter)) { + vo.getRecordList().add(this.convert2TopicRecordVO(record)); + } + + vo.setMaxRecordTimestampUnitMs(Math.max(vo.getMaxRecordTimestampUnitMs(), record.timestamp())); + } + + vo.setTotalRecordCount(vo.getRecordList().size()); + vo.setPartitionConsumedList(new ArrayList<>(consumedVOMap.values())); + if (ValidateUtils.isEmptyList(vo.getPartitionConsumedList())) { + vo.setTotalRecordSizeUnitB(0L); + } else { + vo.setTotalRecordSizeUnitB(vo.getPartitionConsumedList().stream().map(elem -> elem.getRecordSizeUnitB()).reduce(Long::sum).get()); + } + return vo; + } + + private boolean checkMatchFilter(ConsumerRecord consumerRecord, KafkaConsumerFilterDTO filter) { + if (KafkaConsumerFilterEnum.NONE.getCode().equals(filter.getFilterType())) { + return true; + } + + // 包含过滤 + if (KafkaConsumerFilterEnum.CONTAINS.getCode().equals(filter.getFilterType()) + && (!ValidateUtils.isBlank(filter.getFilterCompareKey()) && consumerRecord.key() != null && consumerRecord.key().toString().contains(filter.getFilterCompareKey())) + && (!ValidateUtils.isBlank(filter.getFilterCompareValue()) && consumerRecord.value() != null && consumerRecord.value().toString().contains(filter.getFilterCompareValue()))) { + return true; + } + + // 不包含过滤 + if (KafkaConsumerFilterEnum.NOT_CONTAINS.getCode().equals(filter.getFilterType()) + && (!ValidateUtils.isBlank(filter.getFilterCompareKey()) && (consumerRecord.key() == null || !consumerRecord.key().toString().contains(filter.getFilterCompareKey()))) + && (!ValidateUtils.isBlank(filter.getFilterCompareValue()) && (consumerRecord.value() == null || !consumerRecord.value().toString().contains(filter.getFilterCompareValue())))) { + return true; + } + + // 等于过滤 + if (KafkaConsumerFilterEnum.EQUAL_SIZE.getCode().equals(filter.getFilterType()) + && (!ValidateUtils.isNullOrLessThanZero(filter.getFilterCompareSizeUnitB()) && (consumerRecord.serializedValueSize() + consumerRecord.serializedValueSize()) == filter.getFilterCompareSizeUnitB())) { + return true; + } + + // size大于过滤 + if (KafkaConsumerFilterEnum.ABOVE_SIZE.getCode().equals(filter.getFilterType()) + && (!ValidateUtils.isNullOrLessThanZero(filter.getFilterCompareSizeUnitB()) && (consumerRecord.serializedValueSize() + consumerRecord.serializedValueSize()) > filter.getFilterCompareSizeUnitB())) { + return true; + } + + // size小于过滤 + if (KafkaConsumerFilterEnum.ABOVE_SIZE.getCode().equals(filter.getFilterType()) + && (!ValidateUtils.isNullOrLessThanZero(filter.getFilterCompareSizeUnitB()) && (consumerRecord.serializedValueSize() + consumerRecord.serializedValueSize()) < filter.getFilterCompareSizeUnitB())) { + return true; + } + + return false; + } + + private TopicRecordVO convert2TopicRecordVO(ConsumerRecord consumerRecord) { + TopicRecordVO vo = new TopicRecordVO(); + vo.setTopicName(consumerRecord.topic()); + vo.setPartitionId(consumerRecord.partition()); + vo.setOffset(consumerRecord.offset()); + vo.setTimestampUnitMs(consumerRecord.timestamp()); + vo.setHeaderList(new ArrayList<>()); + for (Header header: consumerRecord.headers()) { + vo.getHeaderList().add(new RecordHeaderKS(header.key(), new String(header.value(), StandardCharsets.UTF_8))); + } + + vo.setKey(consumerRecord.key() == null ? null: consumerRecord.key().toString()); + vo.setValue(consumerRecord.value() == null? null: consumerRecord.value().toString()); + return vo; + } +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerDTO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerDTO.java new file mode 100644 index 00000000..f324c8d3 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerDTO.java @@ -0,0 +1,47 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.ClusterTopicDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.util.Properties; + +/** + * @author zengqiao + * @date 20/4/23 + */ +@Data +@EnterpriseTesting +@ApiModel(description="Kafka消费者测试") +public class KafkaConsumerDTO extends ClusterTopicDTO { + @Valid + @NotNull(message = "startFrom不允许为null") + @ApiModelProperty(value = "消费起始位置信息") + private KafkaConsumerStartFromDTO startFrom; + + @Min(value = 1000, message = "maxDurationUnitMs不允许为null，且不能小于1000ms") + @ApiModelProperty(value = "消费结束信息", example = "10000") + private Long maxDurationUnitMs; + + @Valid + @NotNull(message = "filter不允许为null") + @ApiModelProperty(value = "发送消息条数", example = "6") + private KafkaConsumerFilterDTO filter; + + @NotNull(message = "clientProperties不允许为null") + @ApiModelProperty(value = "客户端配置", example = "{}") + private Properties clientProperties; + + @NotNull(message = "recordOperate不允许为空") + @ApiModelProperty(value = "记录操作，仅记录发起的第一次", example = "false") + private Boolean recordOperate; + + @Min(value = 1, message = "maxRecords不允许为null，且不能小于1") + @ApiModelProperty(value = "消费结束信息", example = "100") + private Integer maxRecords; +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerFilterDTO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerFilterDTO.java new file mode 100644 index 00000000..e184709b --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerFilterDTO.java @@ -0,0 +1,34 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO; +import com.xiaojukeji.know.streaming.km.testing.common.enums.KafkaConsumerFilterEnum; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.hibernate.validator.constraints.Range; + +/** + * @author zengqiao + * @date 20/4/23 + */ +@Data +@EnterpriseTesting +@ApiModel(description="Kafka消费者测试") +public class KafkaConsumerFilterDTO extends BaseDTO { + /** + * @see KafkaConsumerFilterEnum + */ + @Range(min = 0, max = 5, message = "filterType最大和最小值必须在[0, 5]之间") + @ApiModelProperty(value = "开始消费位置的类型", example = "2") + private Integer filterType; + + @ApiModelProperty(value = "比较匹配的Key", example = "ks-km") + private String filterCompareKey; + + @ApiModelProperty(value = "比较匹配的Value", example = "ks-km") + private String filterCompareValue; + + @ApiModelProperty(value = "比较匹配的大小", example = "1024") + private Long filterCompareSizeUnitB; +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerStartFromDTO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerStartFromDTO.java new file mode 100644 index 00000000..fa4fb5fc --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerStartFromDTO.java @@ -0,0 +1,40 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO; +import com.xiaojukeji.know.streaming.km.common.bean.dto.partition.PartitionOffsetDTO; +import com.xiaojukeji.know.streaming.km.testing.common.enums.KafkaConsumerStartFromEnum; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.hibernate.validator.constraints.Range; + +import java.util.List; + +/** + * @author zengqiao + * @date 20/4/23 + */ +@Data +@EnterpriseTesting +@ApiModel(description="Kafka消费者测试") +public class KafkaConsumerStartFromDTO extends BaseDTO { + /** + * @see KafkaConsumerStartFromEnum + */ + @Range(min = 0, max = 5, message = "startFromType最大和最小值必须在[0, 5]之间") + @ApiModelProperty(value = "开始消费位置的类型", example = "2") + private Integer startFromType; + + @ApiModelProperty(value = "指定时间戳消费", example = "2453535465") + private Long timestampUnitMs; + + @ApiModelProperty(value = "指定offset消费", example = "[]") + private List offsetList; + + @ApiModelProperty(value = "指定消费组消费", example = "6") + private String consumerGroup; + + @ApiModelProperty(value = "指定从最近多少条开始消费", example = "10") + private Long latestMinusX; +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaProducerDTO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaProducerDTO.java new file mode 100644 index 00000000..0fac1907 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaProducerDTO.java @@ -0,0 +1,46 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.dto; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.ClusterTopicDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.util.List; +import java.util.Properties; + +/** + * @author zengqiao + * @date 20/4/23 + */ +@Data +@EnterpriseTesting +@ApiModel(description="Kafka生产者测试") +public class KafkaProducerDTO extends ClusterTopicDTO { + @ApiModelProperty(value = "消息Key", example = "hello know-streaming key") + private String recordKey; + + @NotNull(message = "recordValue不允许为null") + @ApiModelProperty(value = "消息Value", example = "hello know-streaming value") + private String recordValue; + + @ApiModelProperty(value = "recordHeader, key-value结构", example = "{}") + private Properties recordHeader; + + @Min(value = 1, message = "recordCount不允许为null或者小于0") + @ApiModelProperty(value = "发送消息条数", example = "6") + private Integer recordCount; + + @NotNull(message = "clientProperties不允许为null") + @ApiModelProperty(value = "客户端配置", example = "{}") + private Properties clientProperties; + + @ApiModelProperty(value = "分区ID列表，为空时表示不进行控制", example = "[1, 2, 3]") + private List partitionIdList; + + @NotNull(message = "recordOperate不允许为空") + @ApiModelProperty(value = "记录操作，仅记录发起的第一次", example = "false") + private Boolean recordOperate; +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/BaseTestVO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/BaseTestVO.java new file mode 100644 index 00000000..c29bc7c9 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/BaseTestVO.java @@ -0,0 +1,22 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author zengqiao + * @date 21/8/19 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(description = "用户测试") +@EnterpriseTesting +public class BaseTestVO { + @ApiModelProperty(value="花费时间, 单位ms", example = "1") + protected Long costTimeUnitMs; +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestConsumerVO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestConsumerVO.java new file mode 100644 index 00000000..7e5f0800 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestConsumerVO.java @@ -0,0 +1,33 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * @author zengqiao + * @date 21/8/19 + */ +@Data +@ApiModel(description = "测试消费结果") +@EnterpriseTesting +public class TestConsumerVO extends BaseTestVO { + @ApiModelProperty(value = "消费信息") + private List partitionConsumedList; + + @ApiModelProperty(value = "记录信息") + private List recordList; + + @ApiModelProperty(value = "本次消费到的RecordSize总大小", example = "1234567") + private Long totalRecordSizeUnitB; + + @ApiModelProperty(value = "本次消费到的总消息条数", example = "23") + private Integer totalRecordCount; + + @ApiModelProperty(value = "时间戳最大的消息时间", example = "34335532342") + private Long maxRecordTimestampUnitMs; +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestPartitionConsumedVO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestPartitionConsumedVO.java new file mode 100644 index 00000000..d640a418 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestPartitionConsumedVO.java @@ -0,0 +1,31 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + + +/** + * Topic Offset + * @author zengqiao + * @date 22/03/01 + */ +@Data +@EnterpriseTesting +public class TestPartitionConsumedVO { + @ApiModelProperty(value = "分区ID", example = "1") + private Integer partitionId; + + @ApiModelProperty(value = "分区end-offset", example = "123") + private Long logEndOffset; + + @ApiModelProperty(value = "消费到的offset", example = "23") + private Long consumedOffset; + + @ApiModelProperty(value = "消费到的LogSize", example = "23") + private Long recordSizeUnitB; + + @ApiModelProperty(value = "消费到的消息条数", example = "23") + private Integer recordCount; +} + diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestProducerVO.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestProducerVO.java new file mode 100644 index 00000000..21404e24 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/vo/TestProducerVO.java @@ -0,0 +1,33 @@ +package com.xiaojukeji.know.streaming.km.testing.common.bean.vo; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author zengqiao + * @date 21/8/19 + */ +@Data +@NoArgsConstructor +@ApiModel(description = "测试生产结果") +@EnterpriseTesting +public class TestProducerVO extends BaseTestVO { + @ApiModelProperty(value = "数据", example = "1") + private Integer partitionId; + + @ApiModelProperty(value = "数据", example = "123") + private Long offset; + + @ApiModelProperty(value = "数据", example = "12321231321231") + private Long timestampUnitMs; + + public TestProducerVO(Long costTimeUnitMs, Integer partitionId, Long offset, Long timestampUnitMs) { + super(costTimeUnitMs); + this.partitionId = partitionId; + this.offset = offset; + this.timestampUnitMs = timestampUnitMs; + } +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerFilterEnum.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerFilterEnum.java new file mode 100644 index 00000000..f7f94d19 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerFilterEnum.java @@ -0,0 +1,35 @@ +package com.xiaojukeji.know.streaming.km.testing.common.enums; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import lombok.Getter; + +/** + * @author zengqiao + * @date 22/02/25 + */ +@Getter +@EnterpriseTesting +public enum KafkaConsumerFilterEnum { + NONE(0, "无"), + + CONTAINS(1, "包含"), + + NOT_CONTAINS(2, "不包含"), + + EQUAL_SIZE(3, "size等于"), + + ABOVE_SIZE(4, "size大于"), + + UNDER_SIZE(5, "size小于"), + + ; + + private final Integer code; + + private final String message; + + KafkaConsumerFilterEnum(Integer code, String message) { + this.code = code; + this.message = message; + } +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerStartFromEnum.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerStartFromEnum.java new file mode 100644 index 00000000..17d15ead --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerStartFromEnum.java @@ -0,0 +1,35 @@ +package com.xiaojukeji.know.streaming.km.testing.common.enums; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import lombok.Getter; + +/** + * @author zengqiao + * @date 22/02/25 + */ +@Getter +@EnterpriseTesting +public enum KafkaConsumerStartFromEnum { + LATEST(0, "最新位置开始消费"), + + EARLIEST(1, "最旧位置开始消费"), + + PRECISE_TIMESTAMP(2, "指定时间开始消费"), + + PRECISE_OFFSET(3, "指定位置开始消费"), + + CONSUMER_GROUP(4, "指定消费组进行消费"), + + LATEST_MINUS_X_OFFSET(5, "近X条数据开始消费"), + + ; + + private final Integer code; + + private final String message; + + KafkaConsumerStartFromEnum(Integer code, String message) { + this.code = code; + this.message = message; + } +} diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/package-info.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/package-info.java new file mode 100644 index 00000000..cb957681 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/package-info.java @@ -0,0 +1,7 @@ +/** + * 生产消费测试 相关功能模块 + */ +@EnterpriseTesting +package com.xiaojukeji.know.streaming.km.testing.common; + +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; diff --git a/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/rest/KafkaClientController.java b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/rest/KafkaClientController.java new file mode 100644 index 00000000..01be1100 --- /dev/null +++ b/km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/rest/KafkaClientController.java @@ -0,0 +1,46 @@ +package com.xiaojukeji.know.streaming.km.testing.rest; + +import com.didiglobal.logi.security.util.HttpRequestUtil; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseTesting; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.testing.biz.KafkaClientTestManager; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaConsumerDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.dto.KafkaProducerDTO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestConsumerVO; +import com.xiaojukeji.know.streaming.km.testing.common.bean.vo.TestProducerVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * @author zengqiao + * @date 22/02/23 + */ +@EnterpriseTesting +@Api(tags = Constant.SWAGGER_API_TAG_PREFIX + "KafkaClient-相关接口(REST)") +@RestController +@RequestMapping(ApiPrefix.API_V3_PREFIX) +public class KafkaClientController { + @Autowired + private KafkaClientTestManager kafkaClientTestManager; + + @ApiOperation(value = "生产者测试") + @PostMapping(value = "clients/producer") + @ResponseBody + public Result> produceTest(@Validated @RequestBody KafkaProducerDTO dto) { + return kafkaClientTestManager.produceTest(dto, HttpRequestUtil.getOperator()); + } + + @ApiOperation(value = "消费者测试") + @PostMapping(value = "clients/consumer") + @ResponseBody + public Result consumeTest(@Validated @RequestBody KafkaConsumerDTO dto) { + return kafkaClientTestManager.consumeTest(dto, HttpRequestUtil.getOperator()); + } +} diff --git a/km-extends/km-license/pom.xml b/km-extends/km-license/pom.xml new file mode 100644 index 00000000..e1f9ebc3 --- /dev/null +++ b/km-extends/km-license/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + com.xiaojukeji.kafka + km-license + ${km.revision} + jar + + + km + com.xiaojukeji.kafka + ${km.revision} + ../../pom.xml + + + + + com.xiaojukeji.kafka + km-common + ${project.parent.version} + + + com.xiaojukeji.kafka + km-persistence + ${project.parent.version} + + + com.xiaojukeji.kafka + km-core + ${project.parent.version} + + + + javax.servlet + javax.servlet-api + + + \ No newline at end of file diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseInterceptor.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseInterceptor.java new file mode 100644 index 00000000..5c6ee2a4 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseInterceptor.java @@ -0,0 +1,69 @@ +package com.xiaojukeji.know.streaming.km.license; + +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; +import com.xiaojukeji.know.streaming.km.license.service.LicenseService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.HandlerInterceptor; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.io.OutputStream; + +import static com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix.API_V3_PREFIX; + +@Component +public class LicenseInterceptor implements HandlerInterceptor { + private static final ILog LOGGER = LogFactory.getLog(LicenseInterceptor.class); + + private static final String PHYSICAL_CLUSTER_URL = API_V3_PREFIX + "physical-clusters"; + private static final String UTF_8 = "utf-8"; + + @Autowired + private LicenseService licenseService; + + /** + * 拦截预处理 + * @return boolean false:拦截, 不向下执行, true:放行 + */ + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + if (PHYSICAL_CLUSTER_URL.equals( request.getRequestURI() ) && + "POST".equals( request.getMethod() )) { + + Result result = licenseService.addClusterLimit(); + if (result.failed()) { + // 如果出错，构造错误信息 + OutputStream out = null; + try { + response.setCharacterEncoding(UTF_8); + response.setContentType("text/json"); + out = response.getOutputStream(); + out.write(ConvertUtil.obj2Json(result).getBytes(UTF_8)); + out.flush(); + } catch (IOException e) { + LOGGER.error( "method=preHandle||msg=physical-clusters add exception! ", e); + } finally { + try { + if (out != null) { + out.close(); + } + } catch (IOException e) { + LOGGER.error( "method=preHandle||msg=outputStream close exception! ", e); + } + } + + // 拒绝向下执行 + return false; + } + } + + // 未达到限制，继续后续的执行 + return true; + } +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseWebConfig.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseWebConfig.java new file mode 100644 index 00000000..54b0d9a3 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/LicenseWebConfig.java @@ -0,0 +1,24 @@ +package com.xiaojukeji.know.streaming.km.license; + +import com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +/** + * @author didi + */ +@Configuration +public class LicenseWebConfig implements WebMvcConfigurer { + @Autowired + private LicenseInterceptor licenseInterceptor; + + @Override + public void addInterceptors(InterceptorRegistry registry) { + // 会进行拦截的接口 + registry.addInterceptor(licenseInterceptor).addPathPatterns(ApiPrefix.API_PREFIX + "**"); + } + +} + diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicense.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicense.java new file mode 100644 index 00000000..d378d461 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicense.java @@ -0,0 +1,11 @@ +package com.xiaojukeji.know.streaming.km.license.bean; + +import lombok.Data; + +/** + * @author didi + */ +@Data +public class KmLicense { + private int clusters; +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicenseUsageDetail.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicenseUsageDetail.java new file mode 100644 index 00000000..ad801a18 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/KmLicenseUsageDetail.java @@ -0,0 +1,24 @@ +package com.xiaojukeji.know.streaming.km.license.bean; + +import lombok.Data; + +import java.util.List; + +/** + * @author didi + */ +@Data +public class KmLicenseUsageDetail { + /** + * 上报的 ks 的节点 + */ + private String host; + /** + * 上报的 ks 的集群的所有的节点 + */ + private List hosts; + /** + * 上报的 ks 集群中 kafka 集群数量 + */ + private int clusters; +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseInfo.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseInfo.java new file mode 100644 index 00000000..22398c96 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseInfo.java @@ -0,0 +1,34 @@ +package com.xiaojukeji.know.streaming.km.license.bean; + +import lombok.Data; + +/** + * @author didi + */ +@Data +public class LicenseInfo { + /** + * + */ + private int status; + + /** + * license 过期时间，单位秒 + */ + private Long expiredDate; + + /** + * + */ + private String app; + + /** + * + */ + private String type; + + /** + * + */ + private T info; +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseResult.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseResult.java new file mode 100644 index 00000000..86ff9ab5 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseResult.java @@ -0,0 +1,12 @@ +package com.xiaojukeji.know.streaming.km.license.bean; + +import lombok.Data; + +/** + * @author didi + */ +@Data +public class LicenseResult { + String err; + T reply; +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseUsage.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseUsage.java new file mode 100644 index 00000000..b7808239 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/bean/LicenseUsage.java @@ -0,0 +1,24 @@ +package com.xiaojukeji.know.streaming.km.license.bean; + +import lombok.Data; + +/** + * @author didi + */ +@Data +public class LicenseUsage { + /** + * 上报时间戳 + */ + private Long timeStamp; + + /** + * uuid + */ + private String uuid; + + /** + * 业务数据 + */ + private String data; +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/controller/LicenseController.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/controller/LicenseController.java new file mode 100644 index 00000000..e5336ffe --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/controller/LicenseController.java @@ -0,0 +1,27 @@ +package com.xiaojukeji.know.streaming.km.license.controller; + +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix; +import com.xiaojukeji.know.streaming.km.license.service.LicenseService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; + +/** + * @author didi + */ +@RestController +@RequestMapping(ApiPrefix.API_V3_PREFIX) +public class LicenseController { + + @Autowired + private LicenseService licenseService; + + @GetMapping(value = "license") + @ResponseBody + public Result check() { + return licenseService.check(); + } +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/LicenseService.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/LicenseService.java new file mode 100644 index 00000000..4239ffc9 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/LicenseService.java @@ -0,0 +1,18 @@ +package com.xiaojukeji.know.streaming.km.license.service; + +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; + +public interface LicenseService { + + /** + * 是否达到了 license 现在的集群数量 + * @return + */ + Result addClusterLimit(); + + /** + * 校验 license 是否通过 + * @return + */ + Result check(); +} diff --git a/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/impl/LicenseServiceImpl.java b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/impl/LicenseServiceImpl.java new file mode 100644 index 00000000..608f8760 --- /dev/null +++ b/km-extends/km-license/src/main/java/com/xiaojukeji/know/streaming/km/license/service/impl/LicenseServiceImpl.java @@ -0,0 +1,253 @@ +package com.xiaojukeji.know.streaming.km.license.service.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.component.RestTool; +import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; +import com.xiaojukeji.know.streaming.km.common.utils.NetUtils; +import com.xiaojukeji.know.streaming.km.common.utils.Tuple; +import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService; +import com.xiaojukeji.know.streaming.km.core.service.km.KmNodeService; +import com.xiaojukeji.know.streaming.km.license.service.LicenseService; +import com.xiaojukeji.know.streaming.km.license.bean.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class LicenseServiceImpl implements LicenseService { + private static final ILog LOGGER = LogFactory.getLog(LicenseServiceImpl.class); + + private static final String LICENSE_INFO_URL = "/api/license/info"; + private static final String LICENSE_USAGE_URL = "/api/license/usage"; + + private static final String LICENSE_HEADER_TOKEN = "x-l-token"; + private static final String LICENSE_HEADER_APP = "x-l-app-name"; + private static final String LICENSE_HEADER_SIGNATURE = "x-l-signature"; + + private static final int FAILED_NO_LICENSE = 1000000000; + private static final int FAILED_LICENSE_EXPIRE = 1000000001; + private static final int FAILED_LICENSE_CLUSTER_LIMIT = 1000000002; + + private static final int ONE_HOUR = 60 * 60 * 1000; + + @Value("${license.server}") + private String licenseSrvUrl; + + @Value("${license.signature}") + private String licenseSignature; + + @Value("${license.token}") + private String licenseToken; + + @Value("${license.app-name}") + private String appName; + + @Autowired + private KmNodeService kmNodeService; + + @Autowired + private ClusterPhyService clusterPhyService; + + @Autowired + private RestTool restTool; + + private LicenseInfo kmLicense; + + private List licenseUsages = new ArrayList<>(); + + @Override + public Result addClusterLimit() { + //对 LicenseUsage 按照时间挫，从小到大排序，即最新的在最后面 + licenseUsages.sort((o1, o2) -> o1.getTimeStamp() < o2.getTimeStamp() ? 1 : -1); + + List details = licenseUsages.stream() + .map(l -> JSON.parseObject(l.getData(), KmLicenseUsageDetail.class)) + .collect(Collectors.toList()); + + if(CollectionUtils.isEmpty(details)){return Result.buildSuc();} + + //Tuple.v1 : ks cluster hosts + //Tuple.v2 : ks 集群管理的 kafka 集群个数 + List, Integer>> ksClusterHostsList = new ArrayList<>(); + ksClusterHostsList.add(new Tuple<>(details.get(0).getHosts(), details.get(0).getClusters())); + + //根据 hosts 是否有交集，来获取 ks 的集群列表 + for(KmLicenseUsageDetail detail : details){ + for(Tuple, Integer> tuple : ksClusterHostsList){ + if(isListIntersection(tuple.getV1(), detail.getHosts())){ + tuple.setV1(detail.getHosts()); + tuple.setV2(detail.getClusters()); + }else { + ksClusterHostsList.add(new Tuple<>(detail.getHosts(), detail.getClusters())); + } + } + } + + LOGGER.debug("method=addClusterLimit||details={}||ksClusterHostsList={}", + JSON.toJSONString(details), JSON.toJSONString(ksClusterHostsList)); + + //计算索引 ks 集群管理的 kafka 集群总个数 + final int[] totalKafkaClusterNus = {0}; + ksClusterHostsList.stream().forEach(l -> totalKafkaClusterNus[0] += l.getV2() ); + + if(null == kmLicense) { + return Result.buildFailure(FAILED_NO_LICENSE, "无法获取KS的License信息"); + } + + if(kmLicense.getInfo().getClusters() < totalKafkaClusterNus[0]) { + return Result.buildFailure(FAILED_LICENSE_CLUSTER_LIMIT, String.format("KS管理的Kafka集群已达到License限制的%d个集群", kmLicense.getInfo().getClusters())); + } + + return Result.buildSuc(); + } + + /** + * 当前这个接口只做最小限度的校验，即 km-license 模块和 license 信息存在， + * 其他异常情况，如：license-srv 临时挂掉不考虑 + * check 接口返回的异常 code、msg，就在该模块定义，不要放到 ResultStatus 中 + */ + @Override + public Result check() { + if(null == kmLicense){ + return Result.buildFailure(FAILED_NO_LICENSE, "无法获取KS的license信息"); + } + + if(System.currentTimeMillis() > kmLicense.getExpiredDate() * 1000){ + return Result.buildFailure(FAILED_LICENSE_EXPIRE, "当前KS的license已过期"); + } + + return Result.buildSuc(); + } + + @PostConstruct + public void init(){ + syncLicenseInfo(); + } + + /** + * 每10分钟同步一次 + */ + @Scheduled(cron="0 0/10 * * * ?") + public void syncLicenseInfo(){ + try { + saveLicenseUsageInfo(); + + List licenseUsages = listLicenseUsageInfo(); + if(!CollectionUtils.isEmpty(licenseUsages)){ + this.licenseUsages.clear(); + this.licenseUsages.addAll(licenseUsages); + } + + LicenseInfo kmLicense = this.getLicenseInfo(); + if(null != kmLicense){ + this.kmLicense = kmLicense; + } + } catch (Exception e){ + LOGGER.error("method=syncLicenseInfo||msg=exception!", e); + } + } + + /**************************************************** private method ****************************************************/ + + private LicenseInfo getLicenseInfo(){ + String url = licenseSrvUrl + LICENSE_INFO_URL; + LicenseResult ret = restTool.getForObject( + url, genHeaders(), new TypeReference>(){}); + + LOGGER.debug("method=getLicenseInfo||url={}||ret={}", url, JSON.toJSONString(ret)); + + if(!StringUtils.isEmpty(ret.getErr())){ + return null; + } + + byte[] encrypted = Base64.getDecoder().decode(ret.getReply().getBytes(StandardCharsets.UTF_8)); + LicenseInfo info = JSON.parseObject( + new String(encrypted), + new TypeReference>(){} + ); + + return info; + } + + private List listLicenseUsageInfo(){ + String url = licenseSrvUrl + LICENSE_USAGE_URL; + LicenseResult> ret = restTool.getForObject( + url, genHeaders(), new TypeReference>>(){}); + + LOGGER.debug("method=listLicenseUsageInfo||url={}||ret={}", url, JSON.toJSONString(ret)); + + if(!StringUtils.isEmpty(ret.getErr())){ + return new ArrayList<>(); + } + + List licenseUsages = ret.getReply(); + if(!CollectionUtils.isEmpty(licenseUsages)){ + long now = System.currentTimeMillis(); + + return licenseUsages.stream() + .filter(l -> l.getTimeStamp() + 6 * ONE_HOUR > now) + .collect(Collectors.toList()); + } + + return new ArrayList<>(); + } + + private boolean saveLicenseUsageInfo(){ + String host = NetUtils.localHost(); + + KmLicenseUsageDetail detail = new KmLicenseUsageDetail(); + detail.setHost(host); + detail.setHosts(kmNodeService.listKmHosts()); + detail.setClusters(clusterPhyService.listAllClusters().size()); + + LicenseUsage licenseUsage = new LicenseUsage(); + licenseUsage.setTimeStamp(System.currentTimeMillis()); + licenseUsage.setUuid(CommonUtils.getMD5(host)); + licenseUsage.setData(JSON.toJSONString(detail)); + + Map param = new HashMap<>(); + param.put("usageSecret", Base64.getEncoder().encodeToString(JSON.toJSONString(licenseUsage).getBytes(StandardCharsets.UTF_8))); + + String url = licenseSrvUrl + LICENSE_USAGE_URL; + LicenseResult ret = restTool.putForObject(url, genHeaders(), JSON.toJSONString(param), LicenseResult.class); + + LOGGER.debug("method=saveLicenseUsageInfo||url={}||ret={}", url, JSON.toJSONString(ret)); + + if(!StringUtils.isEmpty(ret.getErr())){ + return false; + } + + return true; + } + + private HttpHeaders genHeaders(){ + HttpHeaders headers = new HttpHeaders(); + headers.add(LICENSE_HEADER_TOKEN, licenseToken); + headers.add(LICENSE_HEADER_APP, appName); + headers.add(LICENSE_HEADER_SIGNATURE, licenseSignature); + headers.add("content-type", "application/json"); + return headers; + } + + /** + * 两个 list 是否相交，是否有相同的内容 + * @return + */ + private boolean isListIntersection(List l, List r){ + l.retainAll(r); + return !CollectionUtils.isEmpty(l); + } +} diff --git a/km-extends/km-rebalance/pom.xml b/km-extends/km-rebalance/pom.xml new file mode 100644 index 00000000..3a3f3eff --- /dev/null +++ b/km-extends/km-rebalance/pom.xml @@ -0,0 +1,67 @@ + + + + km + com.xiaojukeji.kafka + ${km.revision} + ../../pom.xml + + 4.0.0 + + km-rebalance + + + + org.apache.kafka + kafka-clients + + + org.elasticsearch.client + elasticsearch-rest-client + + + com.fasterxml.jackson.core + jackson-databind + + + com.google.guava + guava + + + org.slf4j + slf4j-api + + + commons-io + commons-io + + + org.apache.commons + commons-lang3 + + + net.sf.jopt-simple + jopt-simple + + + + + + \ No newline at end of file diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/KafkaRebalanceMain.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/KafkaRebalanceMain.java new file mode 100644 index 00000000..13c460bc --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/KafkaRebalanceMain.java @@ -0,0 +1,139 @@ +package com.xiaojukeji.know.streaming.km.rebalance; + + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.HostEnv; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult; +import com.xiaojukeji.know.streaming.km.rebalance.utils.CommandLineUtils; +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import org.apache.commons.io.FileUtils; +import org.apache.kafka.clients.CommonClientConfigs; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +public class KafkaRebalanceMain { + + public void run(OptionSet options) { + try { + BalanceParameter balanceParameter = new BalanceParameter(); + if (options.has("excluded-topics")) { + balanceParameter.setExcludedTopics(options.valueOf("excluded-topics").toString()); + } + if (options.has("offline-brokers")) { + balanceParameter.setOfflineBrokers(options.valueOf("offline-brokers").toString()); + } + if (options.has("disk-threshold")) { + Double diskThreshold = (Double) options.valueOf("disk-threshold"); + balanceParameter.setDiskThreshold(diskThreshold); + } + if (options.has("cpu-threshold")) { + Double cpuThreshold = (Double) options.valueOf("cpu-threshold"); + balanceParameter.setCpuThreshold(cpuThreshold); + } + if (options.has("network-in-threshold")) { + Double networkInThreshold = (Double) options.valueOf("network-in-threshold"); + balanceParameter.setNetworkInThreshold(networkInThreshold); + } + if (options.has("network-out-threshold")) { + Double networkOutThreshold = (Double) options.valueOf("network-out-threshold"); + balanceParameter.setNetworkOutThreshold(networkOutThreshold); + } + if (options.has("balance-brokers")) { + balanceParameter.setBalanceBrokers(options.valueOf("balance-brokers").toString()); + } + if (options.has("topic-leader-threshold")) { + Double topicLeaderThreshold = (Double) options.valueOf("topic-leader-threshold"); + balanceParameter.setTopicLeaderThreshold(topicLeaderThreshold); + } + if (options.has("topic-replica-threshold")) { + Double topicReplicaThreshold = (Double) options.valueOf("topic-replica-threshold"); + balanceParameter.setTopicReplicaThreshold(topicReplicaThreshold); + } + if (options.has("ignored-topics")) { + balanceParameter.setIgnoredTopics(options.valueOf("ignored-topics").toString()); + } + String path = options.valueOf("output-path").toString(); + String goals = options.valueOf("goals").toString(); + balanceParameter.setGoals(Arrays.asList(goals.split(","))); + balanceParameter.setCluster(options.valueOf("cluster").toString()); + Properties kafkaConfig = new Properties(); + kafkaConfig.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, options.valueOf("bootstrap-servers").toString()); + balanceParameter.setKafkaConfig(kafkaConfig); + balanceParameter.setEsRestURL(options.valueOf("es-rest-url").toString()); + balanceParameter.setEsIndexPrefix(options.valueOf("es-index-prefix").toString()); + balanceParameter.setBeforeSeconds((Integer) options.valueOf("before-seconds")); + String envFile = options.valueOf("hardware-env-file").toString(); + String envJson = FileUtils.readFileToString(new File(envFile), "UTF-8"); + List env = new ObjectMapper().readValue(envJson, new TypeReference>() { + }); + balanceParameter.setHardwareEnv(env); + ExecutionRebalance exec = new ExecutionRebalance(); + OptimizerResult optimizerResult = exec.optimizations(balanceParameter); + FileUtils.write(new File(path.concat("/overview.json")), optimizerResult.resultJsonOverview(), "UTF-8"); + FileUtils.write(new File(path.concat("/detailed.json")), optimizerResult.resultJsonDetailed(), "UTF-8"); + FileUtils.write(new File(path.concat("/task.json")), optimizerResult.resultJsonTask(), "UTF-8"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + OptionParser parser = new OptionParser(); + parser.accepts("bootstrap-servers", "Kafka cluster boot server").withRequiredArg().ofType(String.class); + parser.accepts("es-rest-url", "The url of elasticsearch").withRequiredArg().ofType(String.class); + parser.accepts("es-index-prefix", "The Index Prefix of elasticsearch").withRequiredArg().ofType(String.class); + parser.accepts("goals", "Balanced goals include TopicLeadersDistributionGoal,TopicReplicaDistributionGoal,DiskDistributionGoal,NetworkInboundDistributionGoal,NetworkOutboundDistributionGoal").withRequiredArg().ofType(String.class); + parser.accepts("cluster", "Balanced cluster name").withRequiredArg().ofType(String.class); + parser.accepts("excluded-topics", "Topic does not perform data balancing").withOptionalArg().ofType(String.class); + parser.accepts("ignored-topics","Topics that do not contain model calculations").withOptionalArg().ofType(String.class); + parser.accepts("offline-brokers", "Broker does not perform data balancing").withOptionalArg().ofType(String.class); + parser.accepts("balance-brokers", "Balanced brokers list").withOptionalArg().ofType(String.class); + parser.accepts("disk-threshold", "Disk data balance threshold").withOptionalArg().ofType(Double.class); + parser.accepts("topic-leader-threshold","topic leader threshold").withOptionalArg().ofType(Double.class); + parser.accepts("topic-replica-threshold","topic replica threshold").withOptionalArg().ofType(Double.class); + parser.accepts("cpu-threshold", "Cpu utilization balance threshold").withOptionalArg().ofType(Double.class); + parser.accepts("network-in-threshold", "Network inflow threshold").withOptionalArg().ofType(Double.class); + parser.accepts("network-out-threshold", "Network outflow threshold").withOptionalArg().ofType(Double.class); + parser.accepts("before-seconds", "Query es data time").withRequiredArg().ofType(Integer.class); + parser.accepts("hardware-env-file", "Machine environment information includes cpu, disk and network").withRequiredArg().ofType(String.class); + parser.accepts("output-path", "Cluster balancing result file directory").withRequiredArg().ofType(String.class); + OptionSet options = parser.parse(args); + if (args.length == 0) { + CommandLineUtils.printUsageAndDie(parser, "Running parameters need to be configured to perform cluster balancing"); + } + if (!options.has("bootstrap-servers")) { + CommandLineUtils.printUsageAndDie(parser, "bootstrap-servers cannot be empty"); + } + if (!options.has("es-rest-url")) { + CommandLineUtils.printUsageAndDie(parser, "es-rest-url cannot be empty"); + } + if (!options.has("es-index-prefix")) { + CommandLineUtils.printUsageAndDie(parser, "es-index-prefix cannot be empty"); + } + if (!options.has("goals")) { + CommandLineUtils.printUsageAndDie(parser, "goals cannot be empty"); + } + if (!options.has("cluster")) { + CommandLineUtils.printUsageAndDie(parser, "cluster name cannot be empty"); + } + if (!options.has("before-seconds")) { + CommandLineUtils.printUsageAndDie(parser, "before-seconds cannot be empty"); + } + if (!options.has("hardware-env-file")) { + CommandLineUtils.printUsageAndDie(parser, "hardware-env-file cannot be empty"); + } + if (!options.has("output-path")) { + CommandLineUtils.printUsageAndDie(parser, "output-path cannot be empty"); + } + KafkaRebalanceMain rebalanceMain = new KafkaRebalanceMain(); + rebalanceMain.run(options); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/exception/OptimizationFailureException.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/exception/OptimizationFailureException.java new file mode 100644 index 00000000..ecc08f1e --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/exception/OptimizationFailureException.java @@ -0,0 +1,15 @@ +package com.xiaojukeji.know.streaming.km.rebalance.exception; + +public class OptimizationFailureException extends Exception { + public OptimizationFailureException(String message, Throwable cause) { + super(message, cause); + } + + public OptimizationFailureException(String message) { + super(message); + } + + public OptimizationFailureException(Throwable cause) { + super(cause); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/ExecutionRebalance.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/ExecutionRebalance.java new file mode 100644 index 00000000..dd24e99a --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/ExecutionRebalance.java @@ -0,0 +1,78 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor; + +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceGoal; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceThreshold; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BrokerBalanceState; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Load; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.GoalOptimizer; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult; +import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + + +public class ExecutionRebalance { + private static final Logger logger = LoggerFactory.getLogger(ExecutionRebalance.class); + + public OptimizerResult optimizations(BalanceParameter balanceParameter) { + Validate.isTrue(StringUtils.isNotBlank(balanceParameter.getCluster()), "cluster is empty"); + Validate.isTrue(balanceParameter.getKafkaConfig() != null, "Kafka config properties is empty"); + Validate.isTrue(balanceParameter.getGoals() != null, "Balance goals is empty"); + Validate.isTrue(StringUtils.isNotBlank(balanceParameter.getEsIndexPrefix()), "EsIndexPrefix is empty"); + Validate.isTrue(StringUtils.isNotBlank(balanceParameter.getEsRestURL()), "EsRestURL is empty"); + Validate.isTrue(balanceParameter.getHardwareEnv() != null, "HardwareEnv is empty"); + logger.info("Cluster balancing start"); + ClusterModel clusterModel = GoalUtils.getInitClusterModel(balanceParameter); + GoalOptimizer optimizer = new GoalOptimizer(); + OptimizerResult optimizerResult = optimizer.optimizations(clusterModel, new OptimizationOptions(balanceParameter)); + logger.info("Cluster balancing completed"); + return optimizerResult; + } + + public static Map getClusterAvgResourcesState(BalanceParameter balanceParameter) { + ClusterModel clusterModel = GoalUtils.getInitClusterModel(balanceParameter); + Load load = clusterModel.load(); + Map avgResource = new HashMap<>(); + avgResource.put(Resource.DISK, load.loadFor(Resource.DISK) / clusterModel.brokers().size()); + avgResource.put(Resource.CPU, load.loadFor(Resource.CPU) / clusterModel.brokers().size()); + avgResource.put(Resource.NW_OUT, load.loadFor(Resource.NW_OUT) / clusterModel.brokers().size()); + avgResource.put(Resource.NW_IN, load.loadFor(Resource.NW_IN) / clusterModel.brokers().size()); + return avgResource; + } + + public static Map getBrokerResourcesBalanceState(BalanceParameter balanceParameter) { + Map balanceState = new HashMap<>(); + ClusterModel clusterModel = GoalUtils.getInitClusterModel(balanceParameter); + double[] clusterAvgResource = clusterModel.avgOfUtilization(); + Map balanceThreshold = GoalUtils.getBalanceThreshold(balanceParameter, clusterAvgResource); + clusterModel.brokers().forEach(i -> { + BrokerBalanceState state = new BrokerBalanceState(); + if (balanceParameter.getGoals().contains(BalanceGoal.DISK.goal())) { + state.setDiskAvgResource(i.load().loadFor(Resource.DISK)); + state.setDiskUtilization(i.utilizationFor(Resource.DISK)); + state.setDiskBalanceState(balanceThreshold.get(BalanceGoal.DISK.goal()).state(i.utilizationFor(Resource.DISK))); + } + if (balanceParameter.getGoals().contains(BalanceGoal.NW_IN.goal())) { + state.setBytesInAvgResource(i.load().loadFor(Resource.NW_IN)); + state.setBytesInUtilization(i.utilizationFor(Resource.NW_IN)); + state.setBytesInBalanceState(balanceThreshold.get(BalanceGoal.NW_IN.goal()).state(i.utilizationFor(Resource.NW_IN))); + } + if (balanceParameter.getGoals().contains(BalanceGoal.NW_OUT.goal())) { + state.setBytesOutAvgResource(i.load().loadFor(Resource.NW_OUT)); + state.setBytesOutUtilization(i.utilizationFor(Resource.NW_OUT)); + state.setBytesOutBalanceState(balanceThreshold.get(BalanceGoal.NW_OUT.goal()).state(i.utilizationFor(Resource.NW_OUT))); + } + balanceState.put(i.id(), state); + }); + return balanceState; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceActionHistory.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceActionHistory.java new file mode 100644 index 00000000..c743e3c3 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceActionHistory.java @@ -0,0 +1,76 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +public class BalanceActionHistory { + //均衡目标 + private String goal; + //均衡动作 + private String actionType; + //均衡Topic + private String topic; + //均衡分区 + private int partition; + //源Broker + private int sourceBrokerId; + //目标Broker + private int destinationBrokerId; + + public String getGoal() { + return goal; + } + + public void setGoal(String goal) { + this.goal = goal; + } + + public String getActionType() { + return actionType; + } + + public void setActionType(String actionType) { + this.actionType = actionType; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getPartition() { + return partition; + } + + public void setPartition(int partition) { + this.partition = partition; + } + + public int getSourceBrokerId() { + return sourceBrokerId; + } + + public void setSourceBrokerId(int sourceBrokerId) { + this.sourceBrokerId = sourceBrokerId; + } + + public int getDestinationBrokerId() { + return destinationBrokerId; + } + + public void setDestinationBrokerId(int destinationBrokerId) { + this.destinationBrokerId = destinationBrokerId; + } + + @Override + public String toString() { + return "BalanceActionHistory{" + + "goal='" + goal + '\'' + + ", actionType='" + actionType + '\'' + + ", topic='" + topic + '\'' + + ", partition='" + partition + '\'' + + ", sourceBrokerId='" + sourceBrokerId + '\'' + + ", destinationBrokerId='" + destinationBrokerId + '\'' + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceDetailed.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceDetailed.java new file mode 100644 index 00000000..4c7b841b --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceDetailed.java @@ -0,0 +1,173 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +public class BalanceDetailed { + private int brokerId; + private String host; + //当前CPU使用率 + private double currentCPUUtilization; + //最新CPU使用率 + private double lastCPUUtilization; + //当前磁盘使用率 + private double currentDiskUtilization; + //最新磁盘使用量 + private double lastDiskUtilization; + //当前网卡入流量 + private double currentNetworkInUtilization; + //最新网卡入流量 + private double lastNetworkInUtilization; + //当前网卡出流量 + private double currentNetworkOutUtilization; + //最新网卡出流量 + private double lastNetworkOutUtilization; + //均衡状态 + private int balanceState = 0; + //迁入磁盘容量 + private double moveInDiskSize; + //迁出磁盘容量 + private double moveOutDiskSize; + //迁入副本数 + private double moveInReplicas; + //迁出副本数 + private double moveOutReplicas; + + public int getBrokerId() { + return brokerId; + } + + public void setBrokerId(int brokerId) { + this.brokerId = brokerId; + } + + public double getCurrentCPUUtilization() { + return currentCPUUtilization; + } + + public void setCurrentCPUUtilization(double currentCPUUtilization) { + this.currentCPUUtilization = currentCPUUtilization; + } + + public double getLastCPUUtilization() { + return lastCPUUtilization; + } + + public void setLastCPUUtilization(double lastCPUUtilization) { + this.lastCPUUtilization = lastCPUUtilization; + } + + public double getCurrentDiskUtilization() { + return currentDiskUtilization; + } + + public void setCurrentDiskUtilization(double currentDiskUtilization) { + this.currentDiskUtilization = currentDiskUtilization; + } + + public double getLastDiskUtilization() { + return lastDiskUtilization; + } + + public void setLastDiskUtilization(double lastDiskUtilization) { + this.lastDiskUtilization = lastDiskUtilization; + } + + public double getCurrentNetworkInUtilization() { + return currentNetworkInUtilization; + } + + public void setCurrentNetworkInUtilization(double currentNetworkInUtilization) { + this.currentNetworkInUtilization = currentNetworkInUtilization; + } + + public double getLastNetworkInUtilization() { + return lastNetworkInUtilization; + } + + public void setLastNetworkInUtilization(double lastNetworkInUtilization) { + this.lastNetworkInUtilization = lastNetworkInUtilization; + } + + public double getCurrentNetworkOutUtilization() { + return currentNetworkOutUtilization; + } + + public void setCurrentNetworkOutUtilization(double currentNetworkOutUtilization) { + this.currentNetworkOutUtilization = currentNetworkOutUtilization; + } + + public double getLastNetworkOutUtilization() { + return lastNetworkOutUtilization; + } + + public void setLastNetworkOutUtilization(double lastNetworkOutUtilization) { + this.lastNetworkOutUtilization = lastNetworkOutUtilization; + } + + public int getBalanceState() { + return balanceState; + } + + public void setBalanceState(int balanceState) { + this.balanceState = balanceState; + } + + public double getMoveInDiskSize() { + return moveInDiskSize; + } + + public void setMoveInDiskSize(double moveInDiskSize) { + this.moveInDiskSize = moveInDiskSize; + } + + public double getMoveOutDiskSize() { + return moveOutDiskSize; + } + + public void setMoveOutDiskSize(double moveOutDiskSize) { + this.moveOutDiskSize = moveOutDiskSize; + } + + public double getMoveInReplicas() { + return moveInReplicas; + } + + public void setMoveInReplicas(double moveInReplicas) { + this.moveInReplicas = moveInReplicas; + } + + public double getMoveOutReplicas() { + return moveOutReplicas; + } + + public void setMoveOutReplicas(double moveOutReplicas) { + this.moveOutReplicas = moveOutReplicas; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + @Override + public String toString() { + return "BalanceDetailed{" + + "brokerId=" + brokerId + + ", host='" + host + '\'' + + ", currentCPUUtilization=" + currentCPUUtilization + + ", lastCPUUtilization=" + lastCPUUtilization + + ", currentDiskUtilization=" + currentDiskUtilization + + ", lastDiskUtilization=" + lastDiskUtilization + + ", currentNetworkInUtilization=" + currentNetworkInUtilization + + ", lastNetworkInUtilization=" + lastNetworkInUtilization + + ", currentNetworkOutUtilization=" + currentNetworkOutUtilization + + ", lastNetworkOutUtilization=" + lastNetworkOutUtilization + + ", balanceState=" + balanceState + + ", moveInDiskSize=" + moveInDiskSize + + ", moveOutDiskSize=" + moveOutDiskSize + + ", moveInReplicas=" + moveInReplicas + + ", moveOutReplicas=" + moveOutReplicas + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceGoal.java new file mode 100644 index 00000000..81515812 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceGoal.java @@ -0,0 +1,20 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +public enum BalanceGoal { + // KM传参时使用 + TOPIC_LEADERS("TopicLeadersDistributionGoal"), + TOPIC_REPLICA("TopicReplicaDistributionGoal"), + DISK("DiskDistributionGoal"), + NW_IN("NetworkInboundDistributionGoal"), + NW_OUT("NetworkOutboundDistributionGoal"); + + private final String goal; + + BalanceGoal(String goal) { + this.goal = goal; + } + + public String goal() { + return goal; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceOverview.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceOverview.java new file mode 100644 index 00000000..48c12217 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceOverview.java @@ -0,0 +1,102 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; + +import java.util.Map; + +public class BalanceOverview { + //任务类型 + private String taskType; + //节点范围 + private String nodeRange; + //总的迁移大小 + private double totalMoveSize; + //topic黑名单 + private String topicBlacklist; + //迁移副本数 + private int moveReplicas; + //迁移Topic + private String moveTopics; + //均衡阈值 + private Map balanceThreshold; + //移除节点 + private String removeNode; + + public String getTaskType() { + return taskType; + } + + public void setTaskType(String taskType) { + this.taskType = taskType; + } + + public String getNodeRange() { + return nodeRange; + } + + public void setNodeRange(String nodeRange) { + this.nodeRange = nodeRange; + } + + public double getTotalMoveSize() { + return totalMoveSize; + } + + public void setTotalMoveSize(double totalMoveSize) { + this.totalMoveSize = totalMoveSize; + } + + public String getTopicBlacklist() { + return topicBlacklist; + } + + public void setTopicBlacklist(String topicBlacklist) { + this.topicBlacklist = topicBlacklist; + } + + public int getMoveReplicas() { + return moveReplicas; + } + + public void setMoveReplicas(int moveReplicas) { + this.moveReplicas = moveReplicas; + } + + public String getMoveTopics() { + return moveTopics; + } + + public void setMoveTopics(String moveTopics) { + this.moveTopics = moveTopics; + } + + public Map getBalanceThreshold() { + return balanceThreshold; + } + + public void setBalanceThreshold(Map balanceThreshold) { + this.balanceThreshold = balanceThreshold; + } + + public String getRemoveNode() { + return removeNode; + } + + public void setRemoveNode(String removeNode) { + this.removeNode = removeNode; + } + + @Override + public String toString() { + return "BalanceOverview{" + + "taskType='" + taskType + '\'' + + ", nodeRange='" + nodeRange + '\'' + + ", totalMoveSize=" + totalMoveSize + + ", topicBlacklist='" + topicBlacklist + '\'' + + ", moveReplicas=" + moveReplicas + + ", moveTopics='" + moveTopics + '\'' + + ", balanceThreshold=" + balanceThreshold + + ", removeNode='" + removeNode + '\'' + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceParameter.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceParameter.java new file mode 100644 index 00000000..78ce53eb --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceParameter.java @@ -0,0 +1,199 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +import java.util.List; +import java.util.Properties; + +public class BalanceParameter { + //集群名称 + private String cluster; + //集群访问配置 + private Properties kafkaConfig; + //ES访问地址 + private String esRestURL; + //ES存储索引前缀 + private String esIndexPrefix; + //均衡目标 + private List goals; + //Topic黑名单，（参与模型计算） + private String excludedTopics = ""; + //忽略的Topic列表，（不参与模型计算） + private String ignoredTopics = ""; + //下线的Broker + private String offlineBrokers = ""; + //需要均衡的Broker + private String balanceBrokers = ""; + //默认Topic副本分布阈值 + private double topicReplicaThreshold = 0.1; + //磁盘浮动阈值 + private double diskThreshold = 0.1; + //CPU浮动阈值 + private double cpuThreshold = 0.1; + //流入浮动阈值 + private double networkInThreshold = 0.1; + //流出浮动阈值 + private double networkOutThreshold = 0.1; + //均衡时间范围 + private int beforeSeconds = 300; + //集群中所有Broker的硬件环境:cpu、disk、bytesIn、bytesOut + private List hardwareEnv; + //最小Leader浮动阈值,不追求绝对平均，避免集群流量抖动 + private double topicLeaderThreshold = 0.1; + + public String getCluster() { + return cluster; + } + + public void setCluster(String cluster) { + this.cluster = cluster; + } + + public String getEsRestURL() { + return esRestURL; + } + + public void setEsRestURL(String esRestURL) { + this.esRestURL = esRestURL; + } + + public List getGoals() { + return goals; + } + + public void setGoals(List goals) { + this.goals = goals; + } + + public String getExcludedTopics() { + return excludedTopics; + } + + public void setExcludedTopics(String excludedTopics) { + this.excludedTopics = excludedTopics; + } + + public String getIgnoredTopics() { + return ignoredTopics; + } + + public void setIgnoredTopics(String ignoredTopics) { + this.ignoredTopics = ignoredTopics; + } + + public double getTopicReplicaThreshold() { + return topicReplicaThreshold; + } + + public void setTopicReplicaThreshold(double topicReplicaThreshold) { + this.topicReplicaThreshold = topicReplicaThreshold; + } + + public double getDiskThreshold() { + return diskThreshold; + } + + public void setDiskThreshold(double diskThreshold) { + this.diskThreshold = diskThreshold; + } + + public double getCpuThreshold() { + return cpuThreshold; + } + + public void setCpuThreshold(double cpuThreshold) { + this.cpuThreshold = cpuThreshold; + } + + public double getNetworkInThreshold() { + return networkInThreshold; + } + + public void setNetworkInThreshold(double networkInThreshold) { + this.networkInThreshold = networkInThreshold; + } + + public double getNetworkOutThreshold() { + return networkOutThreshold; + } + + public void setNetworkOutThreshold(double networkOutThreshold) { + this.networkOutThreshold = networkOutThreshold; + } + + public List getHardwareEnv() { + return hardwareEnv; + } + + public void setHardwareEnv(List hardwareEnv) { + this.hardwareEnv = hardwareEnv; + } + + public String getBalanceBrokers() { + return balanceBrokers; + } + + public void setBalanceBrokers(String balanceBrokers) { + this.balanceBrokers = balanceBrokers; + } + + public Properties getKafkaConfig() { + return kafkaConfig; + } + + public void setKafkaConfig(Properties kafkaConfig) { + this.kafkaConfig = kafkaConfig; + } + + public String getEsIndexPrefix() { + return esIndexPrefix; + } + + public void setEsIndexPrefix(String esIndexPrefix) { + this.esIndexPrefix = esIndexPrefix; + } + + public String getOfflineBrokers() { + return offlineBrokers; + } + + public void setOfflineBrokers(String offlineBrokers) { + this.offlineBrokers = offlineBrokers; + } + + public int getBeforeSeconds() { + return beforeSeconds; + } + + public void setBeforeSeconds(int beforeSeconds) { + this.beforeSeconds = beforeSeconds; + } + + public double getTopicLeaderThreshold() { + return topicLeaderThreshold; + } + + public void setTopicLeaderThreshold(double topicLeaderThreshold) { + this.topicLeaderThreshold = topicLeaderThreshold; + } + + @Override + public String toString() { + return "BalanceParameter{" + + "cluster='" + cluster + '\'' + + ", kafkaConfig=" + kafkaConfig + + ", esRestURL='" + esRestURL + '\'' + + ", esIndexPrefix='" + esIndexPrefix + '\'' + + ", goals=" + goals + + ", excludedTopics='" + excludedTopics + '\'' + + ", offlineBrokers='" + offlineBrokers + '\'' + + ", balanceBrokers='" + balanceBrokers + '\'' + + ", topicReplicaThreshold=" + topicReplicaThreshold + + ", diskThreshold=" + diskThreshold + + ", cpuThreshold=" + cpuThreshold + + ", networkInThreshold=" + networkInThreshold + + ", networkOutThreshold=" + networkOutThreshold + + ", beforeSeconds=" + beforeSeconds + + ", hardwareEnv=" + hardwareEnv + + ", topicLeaderThreshold=" + topicLeaderThreshold + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceTask.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceTask.java new file mode 100644 index 00000000..73ed224b --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceTask.java @@ -0,0 +1,43 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +import java.util.List; + +public class BalanceTask { + private String topic; + private int partition; + //副本分配列表 + private List replicas; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getPartition() { + return partition; + } + + public void setPartition(int partition) { + this.partition = partition; + } + + public List getReplicas() { + return replicas; + } + + public void setReplicas(List replicas) { + this.replicas = replicas; + } + + @Override + public String toString() { + return "BalanceTask{" + + "topic='" + topic + '\'' + + ", partition=" + partition + + ", replicas=" + replicas + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceThreshold.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceThreshold.java new file mode 100644 index 00000000..92091d01 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BalanceThreshold.java @@ -0,0 +1,41 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; + +public class BalanceThreshold { + private final Resource _resource; + private final double _upper; + private final double _lower; + + public BalanceThreshold(Resource resource, double threshold, double avgResource) { + _resource = resource; + _upper = avgResource * (1 + threshold); + _lower = avgResource * (1 - threshold); + } + + public Resource resource() { + return _resource; + } + + public boolean isInRange(double utilization) { + return utilization > _lower && utilization < _upper; + } + + public int state(double utilization) { + if (utilization <= _lower) { + return -1; + } else if (utilization >= _upper) { + return 1; + } + return 0; + } + + @Override + public String toString() { + return "BalanceThreshold{" + + "_resource=" + _resource + + ", _upper=" + _upper + + ", _lower=" + _lower + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BrokerBalanceState.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BrokerBalanceState.java new file mode 100644 index 00000000..11f7cb55 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/BrokerBalanceState.java @@ -0,0 +1,144 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +public class BrokerBalanceState { + //CPU平均资源 + private Double cpuAvgResource; + //CPU资源使用率 + private Double cpuUtilization; + // -1,低于均衡范围 + // 0,均衡范围内 + // 1,高于均衡范围 + private Integer cpuBalanceState; + //磁盘平均资源 + private Double diskAvgResource; + //磁盘资源使用率 + private Double diskUtilization; + //磁盘均衡状态 + private Integer diskBalanceState; + //流入平均资源 + private Double bytesInAvgResource; + //流入资源使用率 + private Double bytesInUtilization; + //流入均衡状态 + private Integer bytesInBalanceState; + //流出平均资源 + private Double bytesOutAvgResource; + //流出资源使用率 + private Double bytesOutUtilization; + //流出均衡状态 + private Integer bytesOutBalanceState; + + public Double getCpuAvgResource() { + return cpuAvgResource; + } + + public void setCpuAvgResource(Double cpuAvgResource) { + this.cpuAvgResource = cpuAvgResource; + } + + public Double getCpuUtilization() { + return cpuUtilization; + } + + public void setCpuUtilization(Double cpuUtilization) { + this.cpuUtilization = cpuUtilization; + } + + public Integer getCpuBalanceState() { + return cpuBalanceState; + } + + public void setCpuBalanceState(Integer cpuBalanceState) { + this.cpuBalanceState = cpuBalanceState; + } + + public Double getDiskAvgResource() { + return diskAvgResource; + } + + public void setDiskAvgResource(Double diskAvgResource) { + this.diskAvgResource = diskAvgResource; + } + + public Double getDiskUtilization() { + return diskUtilization; + } + + public void setDiskUtilization(Double diskUtilization) { + this.diskUtilization = diskUtilization; + } + + public Integer getDiskBalanceState() { + return diskBalanceState; + } + + public void setDiskBalanceState(Integer diskBalanceState) { + this.diskBalanceState = diskBalanceState; + } + + public Double getBytesInAvgResource() { + return bytesInAvgResource; + } + + public void setBytesInAvgResource(Double bytesInAvgResource) { + this.bytesInAvgResource = bytesInAvgResource; + } + + public Double getBytesInUtilization() { + return bytesInUtilization; + } + + public void setBytesInUtilization(Double bytesInUtilization) { + this.bytesInUtilization = bytesInUtilization; + } + + public Integer getBytesInBalanceState() { + return bytesInBalanceState; + } + + public void setBytesInBalanceState(Integer bytesInBalanceState) { + this.bytesInBalanceState = bytesInBalanceState; + } + + public Double getBytesOutAvgResource() { + return bytesOutAvgResource; + } + + public void setBytesOutAvgResource(Double bytesOutAvgResource) { + this.bytesOutAvgResource = bytesOutAvgResource; + } + + public Double getBytesOutUtilization() { + return bytesOutUtilization; + } + + public void setBytesOutUtilization(Double bytesOutUtilization) { + this.bytesOutUtilization = bytesOutUtilization; + } + + public Integer getBytesOutBalanceState() { + return bytesOutBalanceState; + } + + public void setBytesOutBalanceState(Integer bytesOutBalanceState) { + this.bytesOutBalanceState = bytesOutBalanceState; + } + + @Override + public String toString() { + return "BrokerBalanceState{" + + "cpuAvgResource=" + cpuAvgResource + + ", cpuUtilization=" + cpuUtilization + + ", cpuBalanceState=" + cpuBalanceState + + ", diskAvgResource=" + diskAvgResource + + ", diskUtilization=" + diskUtilization + + ", diskBalanceState=" + diskBalanceState + + ", bytesInAvgResource=" + bytesInAvgResource + + ", bytesInUtilization=" + bytesInUtilization + + ", bytesInBalanceState=" + bytesInBalanceState + + ", bytesOutAvgResource=" + bytesOutAvgResource + + ", bytesOutUtilization=" + bytesOutUtilization + + ", bytesOutBalanceState=" + bytesOutBalanceState + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/HostEnv.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/HostEnv.java new file mode 100644 index 00000000..08eaf3d6 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/HostEnv.java @@ -0,0 +1,76 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +public class HostEnv { + //BrokerId + private int id; + //机器IP + private String host; + //机架ID + private String rackId; + //CPU核数 + private int cpu; + //磁盘总容量 + private double disk; + //网卡容量 + private double network; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getRackId() { + return rackId; + } + + public void setRackId(String rackId) { + this.rackId = rackId; + } + + public int getCpu() { + return cpu; + } + + public void setCpu(int cpu) { + this.cpu = cpu; + } + + public double getDisk() { + return disk; + } + + public void setDisk(double disk) { + this.disk = disk; + } + + public double getNetwork() { + return network; + } + + public void setNetwork(double network) { + this.network = network; + } + + @Override + public String toString() { + return "HostEnv{" + + "id=" + id + + ", host='" + host + '\'' + + ", rackId='" + rackId + '\'' + + ", cpu=" + cpu + + ", disk=" + disk + + ", network=" + network + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/OptimizerResult.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/OptimizerResult.java new file mode 100644 index 00000000..d3821b4c --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/OptimizerResult.java @@ -0,0 +1,218 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xiaojukeji.know.streaming.km.rebalance.model.Broker; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ExecutionProposal; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions; +import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils; +import org.apache.kafka.common.TopicPartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +import java.util.*; +import java.util.stream.Collectors; + +public class OptimizerResult { + private static final Logger logger = LoggerFactory.getLogger(OptimizerResult.class); + private Set _proposals; + private final BalanceParameter parameter; + private Set _balanceBrokersBefore; + private Set _balanceBrokersAfter; + private final ClusterModel clusterModel; + private final Map> balanceActionHistory; + private final Map balanceThreshold; + + public OptimizerResult(ClusterModel clusterModel, OptimizationOptions optimizationOptions) { + this.clusterModel = clusterModel; + balanceActionHistory = clusterModel.balanceActionHistory(); + parameter = optimizationOptions.parameter(); + double[] clusterAvgResource = clusterModel.avgOfUtilization(); + balanceThreshold = GoalUtils.getBalanceThreshold(parameter, clusterAvgResource); + } + + /** + * 计划概览 + */ + public BalanceOverview resultOverview() { + BalanceOverview overview = new BalanceOverview(); + overview.setTopicBlacklist(parameter.getExcludedTopics()); + overview.setMoveReplicas(_proposals.size()); + overview.setNodeRange(parameter.getBalanceBrokers()); + overview.setRemoveNode(parameter.getOfflineBrokers()); + Map balanceThreshold = new HashMap<>(); + balanceThreshold.put(Resource.CPU, parameter.getCpuThreshold()); + balanceThreshold.put(Resource.DISK, parameter.getDiskThreshold()); + balanceThreshold.put(Resource.NW_IN, parameter.getNetworkInThreshold()); + balanceThreshold.put(Resource.NW_OUT, parameter.getNetworkOutThreshold()); + overview.setBalanceThreshold(balanceThreshold); + Set moveTopicsSet = _proposals.stream().map(j -> j.tp().topic()).collect(Collectors.toSet()); + String moveTopics = String.join(",", moveTopicsSet); + overview.setMoveTopics(moveTopics); + //Leader切换时不需要进行统计 + double totalMoveSize = _proposals.stream().filter(i -> Integer.max(i.replicasToAdd().size(), i.replicasToRemove().size()) != 0).mapToDouble(ExecutionProposal::partitionSize).sum(); + overview.setTotalMoveSize(totalMoveSize); + return overview; + } + + /** + * 计划明细 + */ + public Map resultDetailed() { + Map details = new HashMap<>(); + _balanceBrokersBefore.forEach(i -> { + BalanceDetailed balanceDetailed = new BalanceDetailed(); + balanceDetailed.setBrokerId(i.id()); + balanceDetailed.setHost(i.host()); + balanceDetailed.setCurrentCPUUtilization(i.utilizationFor(Resource.CPU)); + balanceDetailed.setCurrentDiskUtilization(i.utilizationFor(Resource.DISK)); + balanceDetailed.setCurrentNetworkInUtilization(i.utilizationFor(Resource.NW_IN)); + balanceDetailed.setCurrentNetworkOutUtilization(i.utilizationFor(Resource.NW_OUT)); + details.put(i.id(), balanceDetailed); + }); + Map totalAddReplicaCount = new HashMap<>(); + Map totalAddDataSize = new HashMap<>(); + Map totalRemoveReplicaCount = new HashMap<>(); + Map totalRemoveDataSize = new HashMap<>(); + _proposals.forEach(i -> { + i.replicasToAdd().forEach((k, v) -> { + totalAddReplicaCount.merge(k, v[0], Double::sum); + totalAddDataSize.merge(k, v[1], Double::sum); + }); + i.replicasToRemove().forEach((k, v) -> { + totalRemoveReplicaCount.merge(k, v[0], Double::sum); + totalRemoveDataSize.merge(k, v[1], Double::sum); + }); + }); + _balanceBrokersAfter.forEach(i -> { + BalanceDetailed balanceDetailed = details.get(i.id()); + balanceDetailed.setLastCPUUtilization(i.utilizationFor(Resource.CPU)); + balanceDetailed.setLastDiskUtilization(i.utilizationFor(Resource.DISK)); + balanceDetailed.setLastNetworkInUtilization(i.utilizationFor(Resource.NW_IN)); + balanceDetailed.setLastNetworkOutUtilization(i.utilizationFor(Resource.NW_OUT)); + balanceDetailed.setMoveInReplicas(totalAddReplicaCount.getOrDefault(i.id(), 0.0)); + balanceDetailed.setMoveOutReplicas(totalRemoveReplicaCount.getOrDefault(i.id(), 0.0)); + balanceDetailed.setMoveInDiskSize(totalAddDataSize.getOrDefault(i.id(), 0.0)); + balanceDetailed.setMoveOutDiskSize(totalRemoveDataSize.getOrDefault(i.id(), 0.0)); + for (String str : parameter.getGoals()) { + BalanceThreshold threshold = balanceThreshold.get(str); + if (!threshold.isInRange(i.utilizationFor(threshold.resource()))) { + balanceDetailed.setBalanceState(-1); + break; + } + } + }); + + return details; + } + + /** + * 计划任务 + */ + public List resultTask() { + List balanceTasks = new ArrayList<>(); + _proposals.forEach(proposal -> { + BalanceTask task = new BalanceTask(); + task.setTopic(proposal.tp().topic()); + task.setPartition(proposal.tp().partition()); + List replicas = proposal.newReplicas().stream().map(ReplicaPlacementInfo::brokerId).collect(Collectors.toList()); + task.setReplicas(replicas); + balanceTasks.add(task); + }); + return balanceTasks; + } + + public Map> resultBalanceActionHistory() { + return Collections.unmodifiableMap(balanceActionHistory); + } + + public String resultJsonOverview() { + try { + return new ObjectMapper().writeValueAsString(resultOverview()); + } catch (Exception e) { + logger.error("result overview json process error", e); + } + return "{}"; + } + + public String resultJsonDetailed() { + try { + return new ObjectMapper().writeValueAsString(resultDetailed()); + } catch (Exception e) { + logger.error("result detailed json process error", e); + } + return "{}"; + } + + public String resultJsonTask() { + try { + Map reassign = new HashMap<>(); + reassign.put("partitions", resultTask()); + reassign.put("version", 1); + return new ObjectMapper().writeValueAsString(reassign); + } catch (Exception e) { + logger.error("result task json process error", e); + } + return "{}"; + } + + public List resultTopicChangeHistory() { + List topicChangeHistoryList = new ArrayList<>(); + for (ExecutionProposal proposal : _proposals) { + TopicChangeHistory changeHistory = new TopicChangeHistory(); + changeHistory.setTopic(proposal.tp().topic()); + changeHistory.setPartition(proposal.tp().partition()); + changeHistory.setOldLeader(proposal.oldLeader().brokerId()); + changeHistory.setNewLeader(proposal.newReplicas().get(0).brokerId()); + List balanceBefore = proposal.oldReplicas().stream().map(ReplicaPlacementInfo::brokerId).collect(Collectors.toList()); + List balanceAfter = proposal.newReplicas().stream().map(ReplicaPlacementInfo::brokerId).collect(Collectors.toList()); + changeHistory.setBalanceBefore(balanceBefore); + changeHistory.setBalanceAfter(balanceAfter); + topicChangeHistoryList.add(changeHistory); + } + return topicChangeHistoryList; + } + + public String resultJsonTopicChangeHistory() { + try { + return new ObjectMapper().writeValueAsString(resultTopicChangeHistory()); + } catch (Exception e) { + logger.error("result balance topic change history json process error", e); + } + return "{}"; + } + + public String resultJsonBalanceActionHistory() { + try { + return new ObjectMapper().writeValueAsString(balanceActionHistory); + } catch (Exception e) { + logger.error("result balance action history json process error", e); + } + return "{}"; + } + + public void setBalanceBrokersFormBefore(Set balanceBrokersBefore) { + _balanceBrokersBefore = new HashSet<>(); + balanceBrokersBefore.forEach(i -> { + Broker broker = new Broker(i.rack(), i.id(), i.host(), false, i.capacity()); + broker.load().addLoad(i.load()); + _balanceBrokersBefore.add(broker); + }); + } + + public void setBalanceBrokersFormAfter(Set balanceBrokersAfter) { + _balanceBrokersAfter = balanceBrokersAfter; + } + + public void setExecutionProposal(Set proposals) { + _proposals = proposals; + } + + // test + public ClusterModel clusterModel() { + return clusterModel; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/TopicChangeHistory.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/TopicChangeHistory.java new file mode 100644 index 00000000..692757a5 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/executor/common/TopicChangeHistory.java @@ -0,0 +1,78 @@ +package com.xiaojukeji.know.streaming.km.rebalance.executor.common; + +import java.util.List; + +public class TopicChangeHistory { + //均衡Topic + private String topic; + //均衡分区 + private int partition; + //旧Leader的BrokerID + private int oldLeader; + //均衡前副本分布 + private List balanceBefore; + //新Leader的BrokerID + private int newLeader; + //均衡后副本分布 + private List balanceAfter; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getPartition() { + return partition; + } + + public void setPartition(int partition) { + this.partition = partition; + } + + public int getOldLeader() { + return oldLeader; + } + + public void setOldLeader(int oldLeader) { + this.oldLeader = oldLeader; + } + + public List getBalanceBefore() { + return balanceBefore; + } + + public void setBalanceBefore(List balanceBefore) { + this.balanceBefore = balanceBefore; + } + + public int getNewLeader() { + return newLeader; + } + + public void setNewLeader(int newLeader) { + this.newLeader = newLeader; + } + + public List getBalanceAfter() { + return balanceAfter; + } + + public void setBalanceAfter(List balanceAfter) { + this.balanceAfter = balanceAfter; + } + + @Override + public String toString() { + return "TopicChangeHistory{" + + "topic='" + topic + '\'' + + ", partition='" + partition + '\'' + + ", oldLeader=" + oldLeader + + ", balanceBefore=" + balanceBefore + + ", newLeader=" + newLeader + + ", balanceAfter=" + balanceAfter + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metric.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metric.java new file mode 100644 index 00000000..cd427b9e --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metric.java @@ -0,0 +1,51 @@ +package com.xiaojukeji.know.streaming.km.rebalance.metric; + +/** + * @author leewei + * @date 2022/5/12 + */ +public class Metric { + private String topic; + private int partition; + private double cpu; + private double bytesIn; + private double bytesOut; + private double disk; + + public Metric() { + + } + + public Metric(String topic, int partition, double cpu, double bytesIn, double bytesOut, double disk) { + this.topic = topic; + this.partition = partition; + this.cpu = cpu; + this.bytesIn = bytesIn; + this.bytesOut = bytesOut; + this.disk = disk; + } + + public String topic() { + return topic; + } + + public int partition() { + return partition; + } + + public double cpu() { + return cpu; + } + + public double bytesIn() { + return bytesIn; + } + + public double bytesOut() { + return bytesOut; + } + + public double disk() { + return disk; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/MetricStore.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/MetricStore.java new file mode 100644 index 00000000..a453af21 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/MetricStore.java @@ -0,0 +1,9 @@ +package com.xiaojukeji.know.streaming.km.rebalance.metric; + +/** + * @author leewei + * @date 2022/4/29 + */ +public interface MetricStore { + Metrics getMetrics(String clusterName, int beforeSeconds); +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metrics.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metrics.java new file mode 100644 index 00000000..b0e87b9e --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/Metrics.java @@ -0,0 +1,46 @@ +package com.xiaojukeji.know.streaming.km.rebalance.metric; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Load; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import org.apache.kafka.common.TopicPartition; + +import java.util.*; + +/** + * @author leewei + * @date 2022/4/29 + */ +public class Metrics { + private final Map metricByTopicPartition; + + public Metrics() { + this.metricByTopicPartition = new HashMap<>(); + } + + public void addMetrics(Metric metric) { + TopicPartition topicPartition = new TopicPartition(metric.topic(), metric.partition()); + this.metricByTopicPartition.put(topicPartition, metric); + } + + public List values() { + return Collections.unmodifiableList(new ArrayList<>(this.metricByTopicPartition.values())); + } + + public Metric metric(TopicPartition topicPartition) { + return this.metricByTopicPartition.get(topicPartition); + } + + public Load load(TopicPartition topicPartition) { + Metric metric = this.metricByTopicPartition.get(topicPartition); + if (metric == null) { + return null; + } + Load load = new Load(); + load.setLoad(Resource.CPU, metric.cpu()); + load.setLoad(Resource.NW_IN, metric.bytesIn()); + load.setLoad(Resource.NW_OUT, metric.bytesOut()); + load.setLoad(Resource.DISK, metric.disk()); + + return load; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/elasticsearch/ElasticsearchMetricStore.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/elasticsearch/ElasticsearchMetricStore.java new file mode 100644 index 00000000..fd9a6f90 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/metric/elasticsearch/ElasticsearchMetricStore.java @@ -0,0 +1,109 @@ +package com.xiaojukeji.know.streaming.km.rebalance.metric.elasticsearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xiaojukeji.know.streaming.km.rebalance.metric.Metric; +import com.xiaojukeji.know.streaming.km.rebalance.metric.MetricStore; +import com.xiaojukeji.know.streaming.km.rebalance.metric.Metrics; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Set; +import java.util.TreeSet; + +/** + * @author leewei + * @date 2022/4/29 + */ +public class ElasticsearchMetricStore implements MetricStore { + private final Logger logger = LoggerFactory.getLogger(ElasticsearchMetricStore.class); + private final ObjectMapper objectMapper = new ObjectMapper(); + + private final String hosts; + private final String indexPrefix; + private final String format; + + public ElasticsearchMetricStore(String hosts, String indexPrefix) { + this(hosts, indexPrefix, "yyyy-MM-dd"); + } + + public ElasticsearchMetricStore(String hosts, String indexPrefix, String format) { + this.hosts = hosts; + this.indexPrefix = indexPrefix; + this.format = format; + } + + @Override + public Metrics getMetrics(String clusterName, int beforeSeconds) { + Metrics metrics = new Metrics(); + try { + String metricsQueryJson = IOUtils.resourceToString("/MetricsQuery.json", StandardCharsets.UTF_8); + metricsQueryJson = metricsQueryJson.replaceAll("", Integer.toString(beforeSeconds)) + .replaceAll("", clusterName); + try (RestClient restClient = RestClient.builder(toHttpHosts(this.hosts)).build()) { + Request request = new Request( + "GET", + "/" + indices(beforeSeconds) + "/_search"); + request.setJsonEntity(metricsQueryJson); + logger.debug("Es metrics query for cluster: {} request: {} dsl: {}", clusterName, request, metricsQueryJson); + Response response = restClient.performRequest(request); + if (response.getStatusLine().getStatusCode() == 200) { + JsonNode rootNode = objectMapper.readTree(response.getEntity().getContent()); + JsonNode topics = rootNode.at("/aggregations/by_topic/buckets"); + for (JsonNode topic : topics) { + String topicName = topic.path("key").asText(); + JsonNode partitions = topic.at("/by_partition/buckets"); + for (JsonNode partition : partitions) { + int partitionId = partition.path("key").asInt(); + // double cpu = partition.at("/avg_cpu/value").asDouble(); + double cpu = 0D; + double bytesIn = partition.at("/avg_bytes_in/value").asDouble(); + double bytesOut = partition.at("/avg_bytes_out/value").asDouble(); + double disk = partition.at("/lastest_disk/hits/hits/0/_source/metrics/LogSize").asDouble(); + // add + metrics.addMetrics(new Metric(topicName, partitionId, cpu, bytesIn, bytesOut, disk)); + } + } + } + } + } catch (IOException e) { + throw new IllegalArgumentException("Cannot get metrics of cluster: " + clusterName, e); + } + logger.debug("Es metrics query for cluster: {} result count: {}", clusterName, metrics.values().size()); + return metrics; + } + + private String indices(long beforeSeconds) { + Set indices = new TreeSet<>(); + DateFormat df = new SimpleDateFormat(this.format); + long endTime = System.currentTimeMillis(); + long time = endTime - (beforeSeconds * 1000); + while (time < endTime) { + indices.add(this.indexPrefix + df.format(new Date(time))); + time += 24 * 60 * 60 * 1000; // add 24h + } + indices.add(this.indexPrefix + df.format(new Date(endTime))); + return String.join(",", indices); + } + + private static HttpHost[] toHttpHosts(String url) { + String[] nodes = url.split(","); + HttpHost[] hosts = new HttpHost[nodes.length]; + for (int i = 0; i < nodes.length; i++) { + String [] ipAndPort = nodes[i].split(":"); + hosts[i] = new HttpHost(ipAndPort[0], ipAndPort.length > 1 ? Integer.parseInt(ipAndPort[1]) : 9200); + } + return hosts; + } + +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Broker.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Broker.java new file mode 100644 index 00000000..316d36d5 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Broker.java @@ -0,0 +1,222 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import org.apache.kafka.common.TopicPartition; + +import java.util.*; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * @author leewei + * @date 2022/4/29 + */ +public class Broker implements Comparable { + public static final Broker NONE = new Broker(new Rack("-1"), -1, "localhost", true, new Capacity()); + + private final Rack rack; + private final int id; + private final String host; + private final boolean isOffline; + + private final Set replicas; + private final Set leaderReplicas; + private final Map> topicReplicas; + + private final Load load; + + private final Capacity capacity; + + public Broker(Rack rack, int id, String host, boolean isOffline, Capacity capacity) { + this.rack = rack; + this.id = id; + this.host = host; + this.isOffline = isOffline; + this.replicas = new HashSet<>(); + this.leaderReplicas = new HashSet<>(); + this.topicReplicas = new HashMap<>(); + this.load = new Load(); + this.capacity = capacity; + } + + public Rack rack() { + return rack; + } + + public int id() { + return id; + } + + public String host() { + return host; + } + + public boolean isOffline() { + return isOffline; + } + + public Set replicas() { + return Collections.unmodifiableSet(this.replicas); + } + + public SortedSet sortedReplicasFor(Resource resource, boolean reverse) { + return sortedReplicasFor(null, resource, reverse); + } + + public SortedSet sortedReplicasFor(Predicate filter, Resource resource, boolean reverse) { + Comparator comparator = + Comparator.comparingDouble(r -> r.load().loadFor(resource)) + .thenComparingInt(Replica::hashCode); + if (reverse) + comparator = comparator.reversed(); + SortedSet sortedReplicas = new TreeSet<>(comparator); + if (filter == null) { + sortedReplicas.addAll(this.replicas); + } else { + sortedReplicas.addAll(this.replicas.stream() + .filter(filter).collect(Collectors.toList())); + } + + return sortedReplicas; + } + + public Set leaderReplicas() { + return Collections.unmodifiableSet(this.leaderReplicas); + } + + public Load load() { + return load; + } + + public Capacity capacity() { + return capacity; + } + + public double utilizationFor(Resource resource) { + return this.load.loadFor(resource) / this.capacity.capacityFor(resource); + } + + public double expectedUtilizationAfterAdd(Resource resource, Load loadToChange) { + return (this.load.loadFor(resource) + ((loadToChange == null) ? 0 : loadToChange.loadFor(resource))) + / this.capacity.capacityFor(resource); + } + + public double expectedUtilizationAfterRemove(Resource resource, Load loadToChange) { + return (this.load.loadFor(resource) - ((loadToChange == null) ? 0 : loadToChange.loadFor(resource))) + / this.capacity.capacityFor(resource); + } + + public Replica replica(TopicPartition topicPartition) { + Map replicas = this.topicReplicas.get(topicPartition.topic()); + if (replicas == null) { + return null; + } + return replicas.get(topicPartition.partition()); + } + + void addReplica(Replica replica) { + // Add replica to list of all replicas in the broker. + if (this.replicas.contains(replica)) { + throw new IllegalStateException(String.format("Broker %d already has replica %s", this.id, + replica.topicPartition())); + } + this.replicas.add(replica); + // Add topic replica. + this.topicReplicas.computeIfAbsent(replica.topicPartition().topic(), t -> new HashMap<>()) + .put(replica.topicPartition().partition(), replica); + + // Add leader replica. + if (replica.isLeader()) { + this.leaderReplicas.add(replica); + } + + // Add replica load to the broker load. + this.load.addLoad(replica.load()); + } + + Replica removeReplica(TopicPartition topicPartition) { + Replica replica = replica(topicPartition); + if (replica != null) { + this.replicas.remove(replica); + Map replicas = this.topicReplicas.get(topicPartition.topic()); + if (replicas != null) { + replicas.remove(topicPartition.partition()); + } + if (replica.isLeader()) { + this.leaderReplicas.remove(replica); + } + this.load.subtractLoad(replica.load()); + } + return replica; + } + + Load makeFollower(TopicPartition topicPartition) { + Replica replica = replica(topicPartition); + Load leaderLoadDelta = replica.makeFollower(); + // Remove leadership load from load. + this.load.subtractLoad(leaderLoadDelta); + this.leaderReplicas.remove(replica); + return leaderLoadDelta; + } + + void makeLeader(TopicPartition topicPartition, Load leaderLoadDelta) { + Replica replica = replica(topicPartition); + replica.makeLeader(leaderLoadDelta); + // Add leadership load to load. + this.load.addLoad(leaderLoadDelta); + this.leaderReplicas.add(replica); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Broker broker = (Broker) o; + return id == broker.id; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public int compareTo(Broker o) { + return Integer.compare(id, o.id()); + } + + @Override + public String toString() { + return "Broker{" + + "id=" + id + + ", host='" + host + '\'' + + ", rack=" + rack.id() + + ", replicas=" + replicas + + ", leaderReplicas=" + leaderReplicas + + ", topicReplicas=" + topicReplicas + + ", load=" + load + + ", capacity=" + capacity + + '}'; + } + + public int numLeadersFor(String topicName) { + return (int) replicasOfTopicInBroker(topicName).stream().filter(Replica::isLeader).count(); + } + + public Set topics() { + return topicReplicas.keySet(); + } + + public int numReplicasOfTopicInBroker(String topic) { + Map replicaMap = topicReplicas.get(topic); + return replicaMap == null ? 0 : replicaMap.size(); + } + + public Collection replicasOfTopicInBroker(String topic) { + Map replicaMap = topicReplicas.get(topic); + return replicaMap == null ? Collections.emptySet() : replicaMap.values(); + } + + public Set currentOfflineReplicas() { + return replicas.stream().filter(Replica::isCurrentOffline).collect(Collectors.toSet()); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Capacity.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Capacity.java new file mode 100644 index 00000000..e1db55a9 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Capacity.java @@ -0,0 +1,36 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import java.util.Arrays; + +/** + * @author leewei + * @date 2022/5/9 + */ +public class Capacity { + private final double[] values; + + public Capacity() { + this.values = new double[Resource.values().length]; + } + + public void setCapacity(Resource resource, double capacity) { + this.values[resource.id()] = capacity; + } + + public double capacityFor(Resource resource) { + return this.values[resource.id()]; + } + + public void addCapacity(Capacity capacityToAdd) { + for (Resource resource : Resource.values()) { + this.setCapacity(resource, this.capacityFor(resource) + capacityToAdd.capacityFor(resource)); + } + } + + @Override + public String toString() { + return "Capacity{" + + "values=" + Arrays.toString(values) + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ClusterModel.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ClusterModel.java new file mode 100644 index 00000000..a8f10a2e --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ClusterModel.java @@ -0,0 +1,236 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceActionHistory; +import org.apache.kafka.common.TopicPartition; + +import java.util.*; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * @author leewei + * @date 2022/4/29 + */ +public class ClusterModel { + private final Map racksById; + private final Map brokersById; + private final Map> partitionsByTopic; + private Map> balanceActionHistory; + + public ClusterModel() { + this.racksById = new HashMap<>(); + this.brokersById = new HashMap<>(); + this.partitionsByTopic = new HashMap<>(); + this.balanceActionHistory = new HashMap<>(); + } + + public Rack rack(String rackId) { + return this.racksById.get(rackId); + } + + public Rack addRack(String rackId) { + Rack rack = new Rack(rackId); + this.racksById.putIfAbsent(rackId, rack); + return this.racksById.get(rackId); + } + + public SortedSet brokers() { + return new TreeSet<>(this.brokersById.values()); + } + + public Set topics() { + return this.partitionsByTopic.keySet(); + } + + public SortedSet topic(String name) { + return new TreeSet<>(this.partitionsByTopic.get(name).values()); + } + + public SortedSet sortedBrokersFor(Resource resource, boolean reverse) { + return sortedBrokersFor(null, resource, reverse); + } + + public SortedSet sortedBrokersFor(Predicate filter, Resource resource, boolean reverse) { + Comparator comparator = + Comparator.comparingDouble(b -> b.utilizationFor(resource)) + .thenComparingInt(Broker::id); + if (reverse) + comparator = comparator.reversed(); + SortedSet sortedBrokers = new TreeSet<>(comparator); + if (filter == null) { + sortedBrokers.addAll(this.brokersById.values()); + } else { + sortedBrokers.addAll(this.brokersById.values().stream() + .filter(filter).collect(Collectors.toList())); + } + + return sortedBrokers; + } + + public Load load() { + Load load = new Load(); + for (Broker broker : this.brokersById.values()) { + load.addLoad(broker.load()); + } + return load; + } + + public Capacity capacity() { + Capacity capacity = new Capacity(); + for (Broker broker : this.brokersById.values()) { + capacity.addCapacity(broker.capacity()); + } + return capacity; + } + + public double utilizationFor(Resource resource) { + return load().loadFor(resource) / capacity().capacityFor(resource); + } + + public double[] avgOfUtilization() { + Load load = load(); + Capacity capacity = capacity(); + double[] unils = new double[Resource.values().length]; + for (Resource resource : Resource.values()) { + unils[resource.id()] = load.loadFor(resource) / capacity.capacityFor(resource); + } + return unils; + } + + public Broker broker(int brokerId) { + return this.brokersById.get(brokerId); + } + + public Broker addBroker(String rackId, int brokerId, String host, boolean isOffline, Capacity capacity) { + Rack rack = rack(rackId); + if (rack == null) + throw new IllegalArgumentException("Rack: " + rackId + "is not exists."); + Broker broker = new Broker(rack, brokerId, host, isOffline, capacity); + rack.addBroker(broker); + this.brokersById.put(brokerId, broker); + return broker; + } + + public Replica addReplica(int brokerId, TopicPartition topicPartition, boolean isLeader, Load load) { + return addReplica(brokerId, topicPartition, isLeader, false, load); + } + + public Replica addReplica(int brokerId, TopicPartition topicPartition, boolean isLeader, boolean isOffline, Load load) { + Broker broker = broker(brokerId); + if (broker == null) { + throw new IllegalArgumentException("Broker: " + brokerId + "is not exists."); + } + + Replica replica = new Replica(broker, topicPartition, isLeader, isOffline); + replica.setLoad(load); + // add to broker + broker.addReplica(replica); + + Map partitions = this.partitionsByTopic + .computeIfAbsent(topicPartition.topic(), k -> new HashMap<>()); + + Partition partition = partitions.computeIfAbsent(topicPartition, Partition::new); + if (isLeader) { + partition.addLeader(replica, 0); + } else { + partition.addFollower(replica, partition.replicas().size()); + } + + return replica; + } + + public Replica removeReplica(int brokerId, TopicPartition topicPartition) { + Broker broker = broker(brokerId); + return broker.removeReplica(topicPartition); + } + + public void relocateLeadership(String goal, String actionType, TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) { + relocateLeadership(topicPartition, sourceBrokerId, destinationBrokerId); + addBalanceActionHistory(goal, actionType, topicPartition, sourceBrokerId, destinationBrokerId); + } + + public void relocateLeadership(TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) { + Broker sourceBroker = broker(sourceBrokerId); + Replica sourceReplica = sourceBroker.replica(topicPartition); + if (!sourceReplica.isLeader()) { + throw new IllegalArgumentException("Cannot relocate leadership of partition " + topicPartition + "from broker " + + sourceBrokerId + " to broker " + destinationBrokerId + + " because the source replica isn't leader."); + } + Broker destinationBroker = broker(destinationBrokerId); + Replica destinationReplica = destinationBroker.replica(topicPartition); + if (destinationReplica.isLeader()) { + throw new IllegalArgumentException("Cannot relocate leadership of partition " + topicPartition + "from broker " + + sourceBrokerId + " to broker " + destinationBrokerId + + " because the destination replica is a leader."); + } + Load leaderLoadDelta = sourceBroker.makeFollower(topicPartition); + destinationBroker.makeLeader(topicPartition, leaderLoadDelta); + + Partition partition = this.partitionsByTopic.get(topicPartition.topic()).get(topicPartition); + partition.relocateLeadership(destinationReplica); + } + + public void relocateReplica(String goal, String actionType, TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) { + relocateReplica(topicPartition, sourceBrokerId, destinationBrokerId); + addBalanceActionHistory(goal, actionType, topicPartition, sourceBrokerId, destinationBrokerId); + } + + public void relocateReplica(TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) { + Replica replica = removeReplica(sourceBrokerId, topicPartition); + if (replica == null) { + throw new IllegalArgumentException("Replica is not in the cluster."); + } + Broker destinationBroker = broker(destinationBrokerId); + replica.setBroker(destinationBroker); + destinationBroker.addReplica(replica); + } + + private void addBalanceActionHistory(String goal, String actionType, TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) { + BalanceActionHistory history = new BalanceActionHistory(); + history.setActionType(actionType); + history.setGoal(goal); + history.setTopic(topicPartition.topic()); + history.setPartition(topicPartition.partition()); + history.setSourceBrokerId(sourceBrokerId); + history.setDestinationBrokerId(destinationBrokerId); + this.balanceActionHistory.computeIfAbsent(topicPartition, k -> new ArrayList<>()).add(history); + } + + public Map numLeadersPerTopic(Set topics) { + Map leaderCountByTopicNames = new HashMap<>(); + topics.forEach(topic -> leaderCountByTopicNames.put(topic, partitionsByTopic.get(topic).size())); + return leaderCountByTopicNames; + } + + public Map> getReplicaDistribution() { + Map> replicaDistribution = new HashMap<>(); + for (Map tp : partitionsByTopic.values()) { + tp.values().forEach(i -> { + i.replicas().forEach(j -> replicaDistribution.computeIfAbsent(j.topicPartition(), k -> new ArrayList<>()) + .add(new ReplicaPlacementInfo(j.broker().id(), ""))); + }); + } + return replicaDistribution; + } + + public Replica partition(TopicPartition tp) { + return partitionsByTopic.get(tp.topic()).get(tp).leader(); + } + + public Map getLeaderDistribution() { + Map leaderDistribution = new HashMap<>(); + for (Broker broker : brokersById.values()) { + broker.leaderReplicas().forEach(i -> leaderDistribution.put(i.topicPartition(), new ReplicaPlacementInfo(broker.id(), ""))); + } + return leaderDistribution; + } + + public int numTopicReplicas(String topic) { + return partitionsByTopic.get(topic).size(); + } + + public Map> balanceActionHistory() { + return this.balanceActionHistory; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Load.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Load.java new file mode 100644 index 00000000..6e3ddc09 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Load.java @@ -0,0 +1,42 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import java.util.Arrays; + +/** + * @author leewei + * @date 2022/5/9 + */ +public class Load { + private final double[] values; + + public Load() { + this.values = new double[Resource.values().length]; + } + + public void setLoad(Resource resource, double load) { + this.values[resource.id()] = load; + } + + public double loadFor(Resource resource) { + return this.values[resource.id()]; + } + + public void addLoad(Load loadToAdd) { + for (Resource resource : Resource.values()) { + this.setLoad(resource, this.loadFor(resource) + loadToAdd.loadFor(resource)); + } + } + + public void subtractLoad(Load loadToSubtract) { + for (Resource resource : Resource.values()) { + this.setLoad(resource, this.loadFor(resource) - loadToSubtract.loadFor(resource)); + } + } + + @Override + public String toString() { + return "Load{" + + "values=" + Arrays.toString(values) + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Partition.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Partition.java new file mode 100644 index 00000000..7842875b --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Partition.java @@ -0,0 +1,148 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import org.apache.kafka.common.TopicPartition; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author leewei + * @date 2022/5/11 + */ +public class Partition implements Comparable { + private final TopicPartition topicPartition; + private final List replicas; + + public Partition(TopicPartition topicPartition) { + this.topicPartition = topicPartition; + this.replicas = new ArrayList<>(); + } + + public TopicPartition topicPartition() { + return topicPartition; + } + + public List replicas() { + return replicas; + } + + public Broker originalLeaderBroker() { + return replicas.stream().filter(r -> r.original().isLeader()) + .findFirst().orElseThrow(IllegalStateException::new).broker(); + } + + public Replica leader() { + return replicas.stream() + .filter(Replica::isLeader) + .findFirst() + .orElseThrow(() -> + new IllegalArgumentException("Not found leader of partition " + topicPartition) + ); + } + + public Replica leaderOrNull() { + return replicas.stream() + .filter(Replica::isLeader) + .findFirst() + .orElse(null); + } + + public List followers() { + return replicas.stream() + .filter(r -> !r.isLeader()) + .collect(Collectors.toList()); + } + + Replica replica(long brokerId) { + return replicas.stream() + .filter(r -> r.broker().id() == brokerId) + .findFirst() + .orElseThrow(() -> + new IllegalArgumentException("Requested replica " + brokerId + " is not a replica of partition " + topicPartition) + ); + } + + public boolean isLeaderChanged() { + // return originalLeaderBroker() != this.leader().broker(); + return replicas.stream().anyMatch(Replica::isLeaderChanged); + } + + public boolean isChanged() { + return replicas.stream().anyMatch(Replica::isChanged); + } + + void addLeader(Replica leader, int index) { + if (leaderOrNull() != null) { + throw new IllegalArgumentException(String.format("Partition %s already has a leader replica %s. Cannot " + + "add a new leader replica %s", this.topicPartition, leaderOrNull(), leader)); + } + if (!leader.isLeader()) { + throw new IllegalArgumentException("Inconsistent leadership information. Trying to set " + leader.broker() + + " as the leader for partition " + this.topicPartition + " while the replica is not marked " + + "as a leader."); + } + this.replicas.add(index, leader); + } + + void addFollower(Replica follower, int index) { + if (follower.isLeader()) { + throw new IllegalArgumentException("Inconsistent leadership information. Trying to add follower replica " + + follower + " while it is a leader."); + } + if (!follower.topicPartition().equals(this.topicPartition)) { + throw new IllegalArgumentException("Inconsistent topic partition. Trying to add follower replica " + follower + + " to partition " + this.topicPartition + "."); + } + this.replicas.add(index, follower); + } + + void relocateLeadership(Replica newLeader) { + if (!newLeader.isLeader()) { + throw new IllegalArgumentException("Inconsistent leadership information. Trying to set " + newLeader.broker() + + " as the leader for partition " + this.topicPartition + " while the replica is not marked " + + "as a leader."); + } + int leaderPos = this.replicas.indexOf(newLeader); + swapReplicaPositions(0, leaderPos); + } + + void swapReplicaPositions(int index1, int index2) { + Replica replica1 = this.replicas.get(index1); + Replica replica2 = this.replicas.get(index2); + + this.replicas.set(index2, replica1); + this.replicas.set(index1, replica2); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Partition partition = (Partition) o; + return topicPartition.equals(partition.topicPartition); + } + + @Override + public int hashCode() { + return Objects.hash(topicPartition); + } + + @Override + public String toString() { + return "Partition{" + + "topicPartition=" + topicPartition + + ", replicas=" + replicas + + ", originalLeaderBroker=" + originalLeaderBroker().id() + + ", leader=" + leaderOrNull() + + '}'; + } + + + + @Override + public int compareTo(Partition o) { + return Integer.compare(topicPartition.partition(), o.topicPartition.partition()); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Rack.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Rack.java new file mode 100644 index 00000000..20df81b1 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Rack.java @@ -0,0 +1,67 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import java.util.*; + +/** + * @author leewei + * @date 2022/5/9 + */ +public class Rack { + private final String id; + private final SortedSet brokers; + + public Rack(String id) { + this.id = id; + this.brokers = new TreeSet<>(); + } + + public String id() { + return id; + } + + public SortedSet brokers() { + return Collections.unmodifiableSortedSet(this.brokers); + } + + public Load load() { + Load load = new Load(); + for (Broker broker : this.brokers) { + load.addLoad(broker.load()); + } + return load; + } + + public List replicas() { + List replicas = new ArrayList<>(); + for (Broker broker : this.brokers) { + replicas.addAll(broker.replicas()); + } + return replicas; + } + + Broker addBroker(Broker broker) { + this.brokers.add(broker); + return broker; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Rack rack = (Rack) o; + return Objects.equals(id, rack.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return "Rack{" + + "id='" + id + '\'' + + ", brokers=" + brokers + + '}'; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Replica.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Replica.java new file mode 100644 index 00000000..e546dfa0 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Replica.java @@ -0,0 +1,129 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import org.apache.kafka.common.TopicPartition; + +import java.util.Objects; + +/** + * @author leewei + * @date 2022/4/29 + */ +public class Replica { + private final Load load; + private final Replica original; + private final TopicPartition topicPartition; + private Broker broker; + private boolean isLeader; + private boolean isOffline; + + public Replica(Broker broker, TopicPartition topicPartition, boolean isLeader, boolean isOffline) { + this(broker, topicPartition, isLeader, isOffline, false); + } + + private Replica(Broker broker, TopicPartition topicPartition, boolean isLeader, boolean isOffline, boolean isOriginal) { + if (isOriginal) { + this.original = null; + } else { + this.original = new Replica(broker, topicPartition, isLeader, isOffline, true); + } + this.load = new Load(); + this.topicPartition = topicPartition; + this.broker = broker; + this.isLeader = isLeader; + this.isOffline = isOffline; + } + + public TopicPartition topicPartition() { + return topicPartition; + } + + public Replica original() { + return original; + } + + public Broker broker() { + return broker; + } + + public void setBroker(Broker broker) { + checkOriginal(); + this.broker = broker; + } + + public boolean isLeader() { + return isLeader; + } + + public Load load() { + return load; + } + + void setLoad(Load load) { + checkOriginal(); + this.load.addLoad(load); + } + + Load makeFollower() { + checkOriginal(); + this.isLeader = false; + // TODO cpu recal + Load leaderLoadDelta = new Load(); + leaderLoadDelta.setLoad(Resource.NW_OUT, this.load.loadFor(Resource.NW_OUT)); + this.load.subtractLoad(leaderLoadDelta); + return leaderLoadDelta; + } + + void makeLeader(Load leaderLoadDelta) { + checkOriginal(); + this.isLeader = true; + this.load.addLoad(leaderLoadDelta); + } + + public boolean isLeaderChanged() { + checkOriginal(); + return this.original.isLeader != this.isLeader; + } + + public boolean isChanged() { + checkOriginal(); + return this.original.broker != this.broker || this.original.isLeader != this.isLeader; + } + + private void checkOriginal() { + if (this.original == null) { + throw new IllegalStateException("This is a original replica, this operation is not supported."); + } + } + + @Override + public boolean equals(Object o) { + checkOriginal(); + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Replica replica = (Replica) o; + return topicPartition.equals(replica.topicPartition) && this.original.broker.equals(replica.original.broker); + } + + @Override + public int hashCode() { + checkOriginal(); + return Objects.hash(topicPartition, this.original.broker); + } + + @Override + public String toString() { + checkOriginal(); + return "Replica{" + + "topicPartition=" + topicPartition + + ", originalBroker=" + this.original.broker.id() + + ", broker=" + broker.id() + + ", originalIsLeader=" + this.original.isLeader + + ", isLeader=" + isLeader + + ", load=" + load + + '}'; + } + //todo:副本状态，待考虑 + public boolean isCurrentOffline() { + return isOffline; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ReplicaPlacementInfo.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ReplicaPlacementInfo.java new file mode 100644 index 00000000..b26b9666 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/ReplicaPlacementInfo.java @@ -0,0 +1,48 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import java.util.Objects; + +public class ReplicaPlacementInfo { + private final int _brokerId; + private final String _logdir; + + public ReplicaPlacementInfo(int brokerId, String logdir) { + _brokerId = brokerId; + _logdir = logdir; + } + + public ReplicaPlacementInfo(Integer brokerId) { + this(brokerId, null); + } + + public Integer brokerId() { + return _brokerId; + } + + public String logdir() { + return _logdir; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ReplicaPlacementInfo)) { + return false; + } + ReplicaPlacementInfo info = (ReplicaPlacementInfo) o; + return _brokerId == info._brokerId && Objects.equals(_logdir, info._logdir); + } + + @Override + public int hashCode() { + return Objects.hash(_brokerId, _logdir); + } + + @Override + public String toString() { + if (_logdir == null) { + return String.format("{Broker: %d}", _brokerId); + } else { + return String.format("{Broker: %d, Logdir: %s}", _brokerId, _logdir); + } + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Resource.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Resource.java new file mode 100644 index 00000000..bdca0dc9 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Resource.java @@ -0,0 +1,29 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +/** + * @author leewei + * @date 2022/5/10 + */ +public enum Resource { + CPU("cpu", 0), + NW_IN("bytesIn", 1), + NW_OUT("bytesOut", 2), + DISK("disk", 3); + + private final String resource; + private final int id; + + Resource(String resource, int id) { + this.resource = resource; + this.id = id; + } + + public String resource() { + return this.resource; + } + + public int id() { + return this.id; + } + +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Supplier.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Supplier.java new file mode 100644 index 00000000..fa714ae7 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/model/Supplier.java @@ -0,0 +1,112 @@ +package com.xiaojukeji.know.streaming.km.rebalance.model; + +import com.xiaojukeji.know.streaming.km.rebalance.metric.MetricStore; +import com.xiaojukeji.know.streaming.km.rebalance.metric.Metrics; +import com.xiaojukeji.know.streaming.km.rebalance.metric.elasticsearch.ElasticsearchMetricStore; +import com.xiaojukeji.know.streaming.km.rebalance.utils.MetadataUtils; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author leewei + * @date 2022/5/12 + */ +public class Supplier { + public static Map subConfig(Map config, String prefix, boolean stripPrefix) { + return config.entrySet().stream() + .filter(e -> e.getKey().startsWith(prefix)) + .collect(Collectors.toMap(e -> stripPrefix ? e.getKey().substring(prefix.length()) : e.getKey(), + Map.Entry::getValue)); + } + + public static ClusterModel load(String clusterName, int beforeSeconds, String kafkaBootstrapServer, String esUrls, String esIndexPrefix, Map capacitiesById, Set ignoredTopics) { + Properties kafkaProperties = new Properties(); + kafkaProperties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBootstrapServer); + return load(clusterName, beforeSeconds, kafkaProperties, esUrls, esIndexPrefix, capacitiesById, ignoredTopics); + } + + public static ClusterModel load(String clusterName, int beforeSeconds, Properties kafkaProperties, String esUrls, String esIndexPrefix, Map capacitiesById, Set ignoredTopics) { + MetricStore store = new ElasticsearchMetricStore(esUrls, esIndexPrefix); + Metrics metrics = store.getMetrics(clusterName, beforeSeconds); + return load(kafkaProperties, capacitiesById, metrics, ignoredTopics); + } + + public static ClusterModel load(Properties kafkaProperties, Map capacitiesById, Metrics metrics, Set ignoredTopics) { + ClusterModel model = new ClusterModel(); + Cluster cluster = MetadataUtils.metadata(kafkaProperties); + + // nodes + for (Node node: cluster.nodes()) { + addBroker(node, false, model, capacitiesById); + } + + // replicas + cluster.topics() + .stream() + .filter(topic -> !ignoredTopics.contains(topic)) + .forEach(topic -> { + List partitions = cluster.partitionsForTopic(topic); + for (PartitionInfo partition : partitions) { + // TODO fix ignore no partition leader + if (partition.leader() == null) { + continue; + } + + TopicPartition topicPartition = new TopicPartition(partition.topic(), partition.partition()); + Load leaderLoad = metrics.load(topicPartition); + if (leaderLoad == null) { + if (partition.leader() == null) { + // set empty load + leaderLoad = new Load(); + } else { + throw new IllegalArgumentException("Cannot get leader load of topic partiton: " + topicPartition); + } + } + + // leader nw out + follower nw out + leaderLoad.setLoad(Resource.NW_OUT, + leaderLoad.loadFor(Resource.NW_OUT) + + leaderLoad.loadFor(Resource.NW_IN) * (partition.replicas().length - 1)); + + Load followerLoad = new Load(); + followerLoad.addLoad(leaderLoad); + followerLoad.setLoad(Resource.NW_OUT, 0); + List offlineReplicas = Arrays.asList(partition.offlineReplicas()); + for (Node n : partition.replicas()) { + boolean isLeader = partition.leader() != null && partition.leader().equals(n); + boolean isOffline = offlineReplicas.contains(n); + if (isOffline) { + if (model.broker(n.id()) == null) { + // add offline broker + addBroker(n, true, model, capacitiesById); + } + } + model.addReplica(n.id(), topicPartition, isLeader, isOffline, isLeader ? leaderLoad : followerLoad); + } + } + }); + return model; + } + + private static String rack(Node node) { + return (node.rack() == null || "".equals(node.rack())) ? node.host() : node.rack(); + } + + private static void addBroker(Node node, boolean isOffline, ClusterModel model, Map capacitiesById) { + // rack + Rack rack = model.addRack(rack(node)); + // broker + Capacity capacity = capacitiesById.get(node.id()); + if (capacity == null) + throw new IllegalArgumentException("Cannot get capacity of node: " + node); + + model.addBroker(rack.id(), node.id(), node.host(), isOffline, capacity); + } + +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionAcceptance.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionAcceptance.java new file mode 100644 index 00000000..daddd907 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionAcceptance.java @@ -0,0 +1,5 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +public enum ActionAcceptance { + ACCEPT, REJECT; +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionType.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionType.java new file mode 100644 index 00000000..ad4f6e3e --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ActionType.java @@ -0,0 +1,18 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +public enum ActionType { + REPLICA_MOVEMENT("REPLICA"), + LEADERSHIP_MOVEMENT("LEADER"); +// REPLICA_SWAP("SWAP"); + + private final String _balancingAction; + + ActionType(String balancingAction) { + _balancingAction = balancingAction; + } + + @Override + public String toString() { + return _balancingAction; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/AnalyzerUtils.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/AnalyzerUtils.java new file mode 100644 index 00000000..097a0ea9 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/AnalyzerUtils.java @@ -0,0 +1,73 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Replica; +import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal; +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.common.TopicPartition; + +import java.util.*; +import java.util.stream.Collectors; + +import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance.ACCEPT; + +public class AnalyzerUtils { + + public static Set getSplitTopics(String value) { + if (StringUtils.isBlank(value)) { + return new HashSet<>(); + } + String[] arr = value.split(","); + return Arrays.stream(arr).collect(Collectors.toSet()); + } + + public static Set getSplitBrokers(String value) { + if (StringUtils.isBlank(value)) { + return new HashSet<>(); + } + String[] arr = value.split(","); + return Arrays.stream(arr).map(Integer::valueOf).collect(Collectors.toSet()); + } + + public static Set getDiff(Map> initialReplicaDistribution, + Map initialLeaderDistribution, + ClusterModel optimizedClusterModel) { + Map> finalReplicaDistribution = optimizedClusterModel.getReplicaDistribution(); + if (!initialReplicaDistribution.keySet().equals(finalReplicaDistribution.keySet())) { + throw new IllegalArgumentException("diff distributions with different partitions."); + } + Set diff = new HashSet<>(); + for (Map.Entry> entry : initialReplicaDistribution.entrySet()) { + TopicPartition tp = entry.getKey(); + List initialReplicas = entry.getValue(); + List finalReplicas = finalReplicaDistribution.get(tp); + Replica finalLeader = optimizedClusterModel.partition(tp); + ReplicaPlacementInfo finalLeaderPlacementInfo = new ReplicaPlacementInfo(finalLeader.broker().id(), ""); + if (finalReplicas.equals(initialReplicas) && initialLeaderDistribution.get(tp).equals(finalLeaderPlacementInfo)) { + continue; + } + if (!finalLeaderPlacementInfo.equals(finalReplicas.get(0))) { + int leaderPos = finalReplicas.indexOf(finalLeaderPlacementInfo); + finalReplicas.set(leaderPos, finalReplicas.get(0)); + finalReplicas.set(0, finalLeaderPlacementInfo); + } + double partitionSize = optimizedClusterModel.partition(tp).load().loadFor(Resource.DISK); + diff.add(new ExecutionProposal(tp, partitionSize, initialLeaderDistribution.get(tp), initialReplicas, finalReplicas)); + } + return diff; + } + + public static ActionAcceptance isProposalAcceptableForOptimizedGoals(Set optimizedGoals, + BalancingAction proposal, + ClusterModel clusterModel) { + for (Goal optimizedGoal : optimizedGoals) { + ActionAcceptance actionAcceptance = optimizedGoal.actionAcceptance(proposal, clusterModel); + if (actionAcceptance != ACCEPT) { + return actionAcceptance; + } + } + return ACCEPT; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/BalancingAction.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/BalancingAction.java new file mode 100644 index 00000000..f40665fd --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/BalancingAction.java @@ -0,0 +1,40 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +import org.apache.kafka.common.TopicPartition; + +public class BalancingAction { + private final TopicPartition _tp; + private final Integer _sourceBrokerId; + private final Integer _destinationBrokerId; + private final ActionType _actionType; + + public BalancingAction(TopicPartition tp, + Integer sourceBrokerId, + Integer destinationBrokerId, + ActionType actionType) { + _tp = tp; + _sourceBrokerId = sourceBrokerId; + _destinationBrokerId = destinationBrokerId; + _actionType = actionType; + } + + public Integer sourceBrokerId() { + return _sourceBrokerId; + } + + public Integer destinationBrokerId() { + return _destinationBrokerId; + } + + public ActionType balancingAction() { + return _actionType; + } + + public TopicPartition topicPartition() { + return _tp; + } + + public String topic() { + return _tp.topic(); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ExecutionProposal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ExecutionProposal.java new file mode 100644 index 00000000..ea2f6057 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/ExecutionProposal.java @@ -0,0 +1,72 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo; +import org.apache.kafka.common.TopicPartition; + +import java.util.*; +import java.util.stream.Collectors; + +public class ExecutionProposal { + + private final TopicPartition _tp; + private final double _partitionSize; + private final ReplicaPlacementInfo _oldLeader; + private final List _oldReplicas; + private final List _newReplicas; + private final Set _replicasToAdd; + private final Set _replicasToRemove; + + public ExecutionProposal(TopicPartition tp, + double partitionSize, + ReplicaPlacementInfo oldLeader, + List oldReplicas, + List newReplicas) { + _tp = tp; + _partitionSize = partitionSize; + _oldLeader = oldLeader; + _oldReplicas = oldReplicas == null ? Collections.emptyList() : oldReplicas; + _newReplicas = newReplicas; + Set newBrokerList = _newReplicas.stream().mapToInt(ReplicaPlacementInfo::brokerId).boxed().collect(Collectors.toSet()); + Set oldBrokerList = _oldReplicas.stream().mapToInt(ReplicaPlacementInfo::brokerId).boxed().collect(Collectors.toSet()); + _replicasToAdd = _newReplicas.stream().filter(r -> !oldBrokerList.contains(r.brokerId())).collect(Collectors.toSet()); + _replicasToRemove = _oldReplicas.stream().filter(r -> !newBrokerList.contains(r.brokerId())).collect(Collectors.toSet()); + } + + public TopicPartition tp() { + return _tp; + } + + public double partitionSize() { + return _partitionSize; + } + + public ReplicaPlacementInfo oldLeader() { + return _oldLeader; + } + + public List oldReplicas() { + return _oldReplicas; + } + + public List newReplicas() { + return _newReplicas; + } + + public Map replicasToAdd() { + Map addData = new HashMap<>(); + _replicasToAdd.forEach(i -> { + Double[] total = {1d, _partitionSize}; + addData.put(i.brokerId(), total); + }); + return Collections.unmodifiableMap(addData); + } + + public Map replicasToRemove() { + Map removeData = new HashMap<>(); + _replicasToRemove.forEach(i -> { + Double[] total = {1d, _partitionSize}; + removeData.put(i.brokerId(), total); + }); + return Collections.unmodifiableMap(removeData); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/GoalOptimizer.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/GoalOptimizer.java new file mode 100644 index 00000000..fdce966f --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/GoalOptimizer.java @@ -0,0 +1,48 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal; +import org.apache.kafka.common.TopicPartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * @author leewei + * @date 2022/4/29 + */ +public class GoalOptimizer { + private static final Logger logger = LoggerFactory.getLogger(GoalOptimizer.class); + + public OptimizerResult optimizations(ClusterModel clusterModel, OptimizationOptions optimizationOptions) { + Set optimizedGoals = new HashSet<>(); + OptimizerResult optimizerResult = new OptimizerResult(clusterModel, optimizationOptions); + optimizerResult.setBalanceBrokersFormBefore(clusterModel.brokers()); + Map> initReplicaDistribution = clusterModel.getReplicaDistribution(); + Map initLeaderDistribution = clusterModel.getLeaderDistribution(); + try { + Map goalMap = new HashMap<>(); + ServiceLoader serviceLoader = ServiceLoader.load(Goal.class); + for (Goal goal : serviceLoader) { + goalMap.put(goal.name(), goal); + } + for (String g : optimizationOptions.goals()) { + Goal goal = goalMap.get(g); + if (goal != null) { + logger.info("Start {} balancing", goal.name()); + goal.optimize(clusterModel, optimizedGoals, optimizationOptions); + optimizedGoals.add(goal); + } + } + } catch (Exception e) { + logger.error("Cluster balancing goal error", e); + } + Set proposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, clusterModel); + optimizerResult.setBalanceBrokersFormAfter(clusterModel.brokers()); + optimizerResult.setExecutionProposal(proposals); + return optimizerResult; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/OptimizationOptions.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/OptimizationOptions.java new file mode 100644 index 00000000..5b3809ea --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/OptimizationOptions.java @@ -0,0 +1,60 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer; + +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; + +import java.util.*; + +public class OptimizationOptions { + private final Set _excludedTopics; + private final Set _offlineBrokers; + private final Set _balanceBrokers; + private final Map _resourceBalancePercentage; + private final List _goals; + private final BalanceParameter _parameter; + + public OptimizationOptions(BalanceParameter parameter) { + _parameter = parameter; + _goals = parameter.getGoals(); + _excludedTopics = AnalyzerUtils.getSplitTopics(parameter.getExcludedTopics()); + _offlineBrokers = AnalyzerUtils.getSplitBrokers(parameter.getOfflineBrokers()); + _balanceBrokers = AnalyzerUtils.getSplitBrokers(parameter.getBalanceBrokers()); + _resourceBalancePercentage = new HashMap<>(); + _resourceBalancePercentage.put(Resource.CPU, parameter.getCpuThreshold()); + _resourceBalancePercentage.put(Resource.DISK, parameter.getDiskThreshold()); + _resourceBalancePercentage.put(Resource.NW_IN, parameter.getNetworkInThreshold()); + _resourceBalancePercentage.put(Resource.NW_OUT, parameter.getNetworkOutThreshold()); + } + + public Set excludedTopics() { + return Collections.unmodifiableSet(_excludedTopics); + } + + public Set offlineBrokers() { + return Collections.unmodifiableSet(_offlineBrokers); + } + + public Set balanceBrokers() { + return Collections.unmodifiableSet(_balanceBrokers); + } + + public double resourceBalancePercentageFor(Resource resource) { + return _resourceBalancePercentage.get(resource); + } + + public List goals() { + return Collections.unmodifiableList(_goals); + } + + public double topicReplicaThreshold() { + return _parameter.getTopicReplicaThreshold(); + } + + public BalanceParameter parameter() { + return _parameter; + } + + public double topicLeaderThreshold() { + return _parameter.getTopicLeaderThreshold(); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/AbstractGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/AbstractGoal.java new file mode 100644 index 00000000..0472e674 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/AbstractGoal.java @@ -0,0 +1,129 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Broker; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Replica; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.*; + +import java.util.*; +import java.util.stream.Collectors; + +public abstract class AbstractGoal implements Goal { + + /** + * 均衡算法逻辑处理 + */ + protected abstract void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set optimizedGoals, OptimizationOptions optimizationOptions); + + /** + * 集群列表中的所有Broker循环执行均衡算法 + */ + @Override + public void optimize(ClusterModel clusterModel, Set optimizedGoals, OptimizationOptions optimizationOptions) { + initGoalState(clusterModel, optimizationOptions); + SortedSet brokenBrokers = clusterModel.brokers().stream() + .filter(b -> optimizationOptions.balanceBrokers().isEmpty() + || optimizationOptions.balanceBrokers().contains(b.id())) + .collect(Collectors.toCollection(TreeSet::new)); + + // SortedSet brokenBrokers = clusterModel.brokers(); + + for (Broker broker : brokenBrokers) { + rebalanceForBroker(broker, clusterModel, optimizedGoals, optimizationOptions); + } + } + + protected abstract void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions); + + /** + * 根据已经计算完的均衡副本、候选目标Broker、执行类型来 + * 执行不同的集群模型数据更改操作 + */ + protected Broker maybeApplyBalancingAction(ClusterModel clusterModel, + Replica replica, + Collection candidateBrokers, + ActionType action, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + List eligibleBrokers = eligibleBrokers(replica, candidateBrokers, action, optimizationOptions); + for (Broker broker : eligibleBrokers) { + BalancingAction proposal = new BalancingAction(replica.topicPartition(), replica.broker().id(), broker.id(), action); + //均衡的副本如果存在当前的Broker上则进行下次Broker + if (!legitMove(replica, broker, action)) { + continue; + } + //均衡条件已经满足进行下次Broker + if (!selfSatisfied(clusterModel, proposal)) { + continue; + } + //判断当前均衡操作是否与其他目标冲突,如果冲突则禁止均衡操作 + ActionAcceptance acceptance = AnalyzerUtils.isProposalAcceptableForOptimizedGoals(optimizedGoals, proposal, clusterModel); + if (acceptance == ActionAcceptance.ACCEPT) { + if (action == ActionType.LEADERSHIP_MOVEMENT) { + clusterModel.relocateLeadership(name(), action.toString(), replica.topicPartition(), replica.broker().id(), broker.id()); + } else if (action == ActionType.REPLICA_MOVEMENT) { + clusterModel.relocateReplica(name(), action.toString(), replica.topicPartition(), replica.broker().id(), broker.id()); + } + return broker; + } + } + return null; + } + + /** + * 副本操作合法性判断: + * 1.副本迁移，目的broker不包含移动副本 + * 2.Leader切换，目的broker需要包含切换副本 + */ + private static boolean legitMove(Replica replica, + Broker destinationBroker, ActionType actionType) { + switch (actionType) { + case REPLICA_MOVEMENT: + return destinationBroker.replica(replica.topicPartition()) == null; + case LEADERSHIP_MOVEMENT: + return replica.isLeader() && destinationBroker.replica(replica.topicPartition()) != null; + default: + return false; + } + } + + protected abstract boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action); + + /** + * 候选Broker列表筛选过滤 + */ + public static List eligibleBrokers(Replica replica, + Collection candidates, + ActionType action, + OptimizationOptions optimizationOptions) { + List eligibleBrokers = new ArrayList<>(candidates); + filterOutBrokersExcludedForLeadership(eligibleBrokers, optimizationOptions, replica, action); + filterOutBrokersExcludedForReplicaMove(eligibleBrokers, optimizationOptions, action); + return eligibleBrokers; + } + + /** + * Leader切换，从候选的Broker列表中排除掉excludedBroker + */ + public static void filterOutBrokersExcludedForLeadership(List eligibleBrokers, + OptimizationOptions optimizationOptions, + Replica replica, + ActionType action) { + Set excludedBrokers = optimizationOptions.offlineBrokers(); + if (!excludedBrokers.isEmpty() && (action == ActionType.LEADERSHIP_MOVEMENT || replica.isLeader())) { + eligibleBrokers.removeIf(broker -> excludedBrokers.contains(broker.id())); + } + } + + /** + * 副本迁移，从候选的Broker列表中排除掉excludedBroker + */ + public static void filterOutBrokersExcludedForReplicaMove(List eligibleBrokers, + OptimizationOptions optimizationOptions, + ActionType action) { + Set excludedBrokers = optimizationOptions.offlineBrokers(); + if (!excludedBrokers.isEmpty() && action == ActionType.REPLICA_MOVEMENT) { + eligibleBrokers.removeIf(broker -> excludedBrokers.contains(broker.id())); + } + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/DiskDistributionGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/DiskDistributionGoal.java new file mode 100644 index 00000000..f91179d9 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/DiskDistributionGoal.java @@ -0,0 +1,31 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction; + +/** + * @author leewei + * @date 2022/5/24 + */ +public class DiskDistributionGoal extends ResourceDistributionGoal { + + @Override + protected Resource resource() { + return Resource.DISK; + } + + @Override + public String name() { + return DiskDistributionGoal.class.getSimpleName(); + } + + @Override + public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) { + // Leadership movement won't cause disk utilization change. + return action.balancingAction() == ActionType.LEADERSHIP_MOVEMENT ? ActionAcceptance.ACCEPT : super.actionAcceptance(action, clusterModel); + } + +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/Goal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/Goal.java new file mode 100644 index 00000000..679b8e7c --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/Goal.java @@ -0,0 +1,17 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions; + +import java.util.Set; + +public interface Goal { + + void optimize(ClusterModel clusterModel, Set optimizedGoals, OptimizationOptions optimizationOptions); + + String name(); + + ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel); +} \ No newline at end of file diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkInboundDistributionGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkInboundDistributionGoal.java new file mode 100644 index 00000000..0195caa4 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkInboundDistributionGoal.java @@ -0,0 +1,30 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction; + +/** + * @author leewei + * @date 2022/5/20 + */ +public class NetworkInboundDistributionGoal extends ResourceDistributionGoal { + + @Override + protected Resource resource() { + return Resource.NW_IN; + } + + @Override + public String name() { + return NetworkInboundDistributionGoal.class.getSimpleName(); + } + + @Override + public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) { + // Leadership movement won't cause inbound network utilization change. + return action.balancingAction() == ActionType.LEADERSHIP_MOVEMENT ? ActionAcceptance.ACCEPT : super.actionAcceptance(action, clusterModel); + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkOutboundDistributionGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkOutboundDistributionGoal.java new file mode 100644 index 00000000..eb8809b1 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/NetworkOutboundDistributionGoal.java @@ -0,0 +1,22 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Resource; + +/** + * @author leewei + * @date 2022/5/24 + */ +public class NetworkOutboundDistributionGoal extends ResourceDistributionGoal { + + @Override + protected Resource resource() { + return Resource.NW_OUT; + } + + @Override + public String name() { + return NetworkOutboundDistributionGoal.class.getSimpleName(); + } + +} + diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/ResourceDistributionGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/ResourceDistributionGoal.java new file mode 100644 index 00000000..2eb3f635 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/ResourceDistributionGoal.java @@ -0,0 +1,227 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.*; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Set; +import java.util.SortedSet; + +/** + * @author leewei + * @date 2022/5/20 + */ +public abstract class ResourceDistributionGoal extends AbstractGoal { + private static final Logger logger = LoggerFactory.getLogger(ResourceDistributionGoal.class); + private double balanceUpperThreshold; + private double balanceLowerThreshold; + + @Override + protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) { + double avgUtilization = clusterModel.utilizationFor(resource()); + double balancePercentage = optimizationOptions.resourceBalancePercentageFor(resource()); + this.balanceUpperThreshold = avgUtilization * (1 + balancePercentage); + this.balanceLowerThreshold = avgUtilization * (1 - balancePercentage); + } + + @Override + protected void rebalanceForBroker(Broker broker, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + double utilization = broker.utilizationFor(resource()); + + boolean requireLessLoad = utilization > this.balanceUpperThreshold; + boolean requireMoreLoad = utilization < this.balanceLowerThreshold; + if (!requireMoreLoad && !requireLessLoad) { + return; + } + + // First try leadership movement + if (resource() == Resource.NW_OUT || resource() == Resource.CPU) { + if (requireLessLoad && rebalanceByMovingLoadOut(broker, clusterModel, optimizedGoals, + ActionType.LEADERSHIP_MOVEMENT, optimizationOptions)) { + logger.debug("Successfully balanced {} for broker {} by moving out leaders.", resource(), broker.id()); + requireLessLoad = false; + } + if (requireMoreLoad && rebalanceByMovingLoadIn(broker, clusterModel, optimizedGoals, + ActionType.LEADERSHIP_MOVEMENT, optimizationOptions)) { + logger.debug("Successfully balanced {} for broker {} by moving in leaders.", resource(), broker.id()); + requireMoreLoad = false; + } + } + + boolean balanced = true; + if (requireLessLoad) { + if (!rebalanceByMovingLoadOut(broker, clusterModel, optimizedGoals, + ActionType.REPLICA_MOVEMENT, optimizationOptions)) { + balanced = rebalanceBySwappingLoadOut(broker, clusterModel, optimizedGoals, optimizationOptions); + } + } else if (requireMoreLoad) { + if (!rebalanceByMovingLoadIn(broker, clusterModel, optimizedGoals, + ActionType.REPLICA_MOVEMENT, optimizationOptions)) { + balanced = rebalanceBySwappingLoadIn(broker, clusterModel, optimizedGoals, optimizationOptions); + } + } + if (balanced) { + logger.debug("Successfully balanced {} for broker {} by moving leaders and replicas.", resource(), broker.id()); + } + } + + private boolean rebalanceByMovingLoadOut(Broker broker, + ClusterModel clusterModel, + Set optimizedGoals, + ActionType actionType, + OptimizationOptions optimizationOptions) { + + SortedSet candidateBrokers = sortedCandidateBrokersUnderThreshold(clusterModel, this.balanceUpperThreshold, optimizationOptions, broker, false); + SortedSet replicasToMove = sortedCandidateReplicas(broker, actionType, optimizationOptions, true); + + for (Replica replica : replicasToMove) { + Broker acceptedBroker = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, actionType, optimizedGoals, optimizationOptions); + + if (acceptedBroker != null) { + if (broker.utilizationFor(resource()) < this.balanceUpperThreshold) { + return true; + } + // Remove and reinsert the broker so the order is correct. + // candidateBrokers.remove(acceptedBroker); + candidateBrokers.removeIf(b -> b.id() == acceptedBroker.id()); + if (acceptedBroker.utilizationFor(resource()) < this.balanceUpperThreshold) { + candidateBrokers.add(acceptedBroker); + } + } + } + + return false; + } + + private boolean rebalanceByMovingLoadIn(Broker broker, + ClusterModel clusterModel, + Set optimizedGoals, + ActionType actionType, + OptimizationOptions optimizationOptions) { + SortedSet candidateBrokers = sortedCandidateBrokersOverThreshold(clusterModel, this.balanceLowerThreshold, optimizationOptions, broker, true); + Iterator candidateBrokersIt = candidateBrokers.iterator(); + Broker nextCandidateBroker = null; + while (true) { + Broker candidateBroker; + if (nextCandidateBroker != null) { + candidateBroker = nextCandidateBroker; + nextCandidateBroker = null; + } else if (candidateBrokersIt.hasNext()) { + candidateBroker = candidateBrokersIt.next(); + } else { + break; + } + SortedSet replicasToMove = sortedCandidateReplicas(candidateBroker, actionType, optimizationOptions, true); + + for (Replica replica : replicasToMove) { + Broker acceptedBroker = maybeApplyBalancingAction(clusterModel, replica, Collections.singletonList(broker), actionType, optimizedGoals, optimizationOptions); + if (acceptedBroker != null) { + if (broker.utilizationFor(resource()) > this.balanceLowerThreshold) { + return true; + } + if (candidateBrokersIt.hasNext() || nextCandidateBroker != null) { + if (nextCandidateBroker == null) { + nextCandidateBroker = candidateBrokersIt.next(); + } + if (candidateBroker.utilizationFor(resource()) < nextCandidateBroker.utilizationFor(resource())) { + break; + } + } + } + } + } + + return false; + } + + private boolean rebalanceBySwappingLoadOut(Broker broker, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + return false; + } + + private boolean rebalanceBySwappingLoadIn(Broker broker, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + return false; + } + + private SortedSet sortedCandidateBrokersUnderThreshold(ClusterModel clusterModel, + double utilizationThreshold, + OptimizationOptions optimizationOptions, + Broker excludedBroker, + boolean reverse) { + return clusterModel.sortedBrokersFor( + b -> b.utilizationFor(resource()) < utilizationThreshold + && !excludedBroker.equals(b) + // filter brokers + && (optimizationOptions.balanceBrokers().isEmpty() || optimizationOptions.balanceBrokers().contains(b.id())) + , resource(), reverse); + } + + private SortedSet sortedCandidateBrokersOverThreshold(ClusterModel clusterModel, + double utilizationThreshold, + OptimizationOptions optimizationOptions, + Broker excludedBroker, + boolean reverse) { + return clusterModel.sortedBrokersFor( + b -> b.utilizationFor(resource()) > utilizationThreshold + && !excludedBroker.equals(b) + // filter brokers + && (optimizationOptions.balanceBrokers().isEmpty() || optimizationOptions.balanceBrokers().contains(b.id())) + , resource(), reverse); + } + + private SortedSet sortedCandidateReplicas(Broker broker, + ActionType actionType, + OptimizationOptions optimizationOptions, + boolean reverse) { + return broker.sortedReplicasFor( + // exclude topic + r -> !optimizationOptions.excludedTopics().contains(r.topicPartition().topic()) + && r.load().loadFor(resource()) > 0.0 + // LEADERSHIP_MOVEMENT or NW_OUT is require leader replica + && (actionType != ActionType.LEADERSHIP_MOVEMENT && resource() != Resource.NW_OUT || r.isLeader()) + , resource(), reverse); + } + + protected abstract Resource resource(); + + @Override + protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) { + Broker destinationBroker = clusterModel.broker(action.destinationBrokerId()); + Broker sourceBroker = clusterModel.broker(action.sourceBrokerId()); + Replica sourceReplica = sourceBroker.replica(action.topicPartition()); + + Load loadToChange; + if (action.balancingAction() == ActionType.LEADERSHIP_MOVEMENT) { + Replica destinationReplica = destinationBroker.replica(action.topicPartition()); + Load delta = new Load(); + delta.addLoad(sourceReplica.load()); + delta.subtractLoad(destinationReplica.load()); + loadToChange = delta; + } else { + loadToChange = sourceReplica.load(); + } + double sourceUtilization = sourceBroker.expectedUtilizationAfterRemove(resource(), loadToChange); + double destinationUtilization = destinationBroker.expectedUtilizationAfterAdd(resource(), loadToChange); + + return sourceUtilization >= this.balanceLowerThreshold && destinationUtilization <= this.balanceUpperThreshold; + } + + @Override + public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) { + return this.selfSatisfied(clusterModel, action) ? ActionAcceptance.ACCEPT : ActionAcceptance.REJECT; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicLeadersDistributionGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicLeadersDistributionGoal.java new file mode 100644 index 00000000..83080db7 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicLeadersDistributionGoal.java @@ -0,0 +1,222 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Broker; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Replica; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions; +import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance.ACCEPT; +import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance.REJECT; +import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType.REPLICA_MOVEMENT; +import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType.LEADERSHIP_MOVEMENT; + +public class TopicLeadersDistributionGoal extends AbstractGoal { + private static final Logger logger = LoggerFactory.getLogger(TopicLeadersDistributionGoal.class); + private Map _mustHaveTopicMinLeadersPerBroker; + + /** + * 执行Topic Leader均衡 + */ + @Override + protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set optimizedGoals, OptimizationOptions optimizationOptions) { + moveAwayOfflineReplicas(broker, clusterModel, optimizedGoals, optimizationOptions); + if (_mustHaveTopicMinLeadersPerBroker.isEmpty()) { + return; + } + if (optimizationOptions.offlineBrokers().contains(broker.id())) { + return; + } + + for (String topicName : _mustHaveTopicMinLeadersPerBroker.keySet()) { + maybeMoveLeaderOfTopicToBroker(topicName, broker, clusterModel, optimizedGoals, optimizationOptions); + } + } + + /** + * 初始化均衡条件: + * 1.排除不需要的Broker、Topic + * 2.计算每个Topic在集群中所有Broker的平均分布数量 + */ + @Override + protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) { + _mustHaveTopicMinLeadersPerBroker = new HashMap<>(); + Set excludedTopics = optimizationOptions.excludedTopics(); + Set excludedBrokers = optimizationOptions.offlineBrokers(); + Set mustHaveTopicLeadersPerBroker = GoalUtils.getNotExcludeTopics(clusterModel, excludedTopics); + Map numLeadersByTopicNames = clusterModel.numLeadersPerTopic(mustHaveTopicLeadersPerBroker); + Set allBrokers = GoalUtils.getNotExcludeBrokers(clusterModel, excludedBrokers); + for (String topicName : mustHaveTopicLeadersPerBroker) { + int topicNumLeaders = numLeadersByTopicNames.get(topicName); + int avgLeaders = allBrokers.size() == 0 ? 0 : (int) Math.ceil(topicNumLeaders / (double) allBrokers.size() * (1 + optimizationOptions.topicLeaderThreshold())); + _mustHaveTopicMinLeadersPerBroker.put(topicName, avgLeaders); + } + } + + /** + * 已满足均衡条件判断: + * 1.待操作Broker的副本已下线 + * 2.待操作Broker上Topic Leader数量大于Topic平均分布数量 + */ + @Override + protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) { + Broker sourceBroker = clusterModel.broker(action.sourceBrokerId()); + Replica replicaToBeMoved = sourceBroker.replica(action.topicPartition()); + if (replicaToBeMoved.broker().replica(action.topicPartition()).isCurrentOffline()) { + return action.balancingAction() == REPLICA_MOVEMENT; + } + String topicName = replicaToBeMoved.topicPartition().topic(); + return sourceBroker.numLeadersFor(topicName) > minTopicLeadersPerBroker(topicName); + } + + /** + * 获取Topic在每台Broker上的最小Leader数 + */ + private int minTopicLeadersPerBroker(String topicName) { + return _mustHaveTopicMinLeadersPerBroker.get(topicName); + } + + @Override + public String name() { + return TopicLeadersDistributionGoal.class.getSimpleName(); + } + + /** + * 判断Topic Leader均衡动作是否可以执行 + */ + @Override + public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) { + if (_mustHaveTopicMinLeadersPerBroker.containsKey(action.topic())) { + return ACCEPT; + } + switch (action.balancingAction()) { + case LEADERSHIP_MOVEMENT: + case REPLICA_MOVEMENT: + Replica replicaToBeRemoved = clusterModel.broker(action.sourceBrokerId()).replica(action.topicPartition()); + return doesLeaderRemoveViolateOptimizedGoal(replicaToBeRemoved) ? REJECT : ACCEPT; + default: + throw new IllegalArgumentException("Unsupported balancing action " + action.balancingAction() + " is provided."); + } + } + + /** + * 根据指定的副本判断是否可以执行均衡动作 + */ + private boolean doesLeaderRemoveViolateOptimizedGoal(Replica replicaToBeRemoved) { + if (!replicaToBeRemoved.isLeader()) { + return false; + } + String topic = replicaToBeRemoved.topicPartition().topic(); + if (!_mustHaveTopicMinLeadersPerBroker.containsKey(topic)) { + return false; + } + int topicLeaderCountOnSourceBroker = replicaToBeRemoved.broker().numLeadersFor(topic); + return topicLeaderCountOnSourceBroker <= minTopicLeadersPerBroker(topic); + } + + /** + * 执行具体的均衡逻辑: + * 先通过Leader切换如果还不满足条件则进行副本迁移 + */ + private void maybeMoveLeaderOfTopicToBroker(String topicName, + Broker broker, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + int topicLeaderCount = broker.numLeadersFor(topicName); + //判断Topic在当前Broker上的Leader数量是否超过最小Leader分布 + if (topicLeaderCount >= minTopicLeadersPerBroker(topicName)) { + return; + } + //获取Topic在当前Broker上的所有follower副本 + List followerReplicas = broker.replicas().stream().filter(i -> !i.isLeader() && i.topicPartition().topic().equals(topicName)).collect(Collectors.toList()); + for (Replica followerReplica : followerReplicas) { + //根据follower副本信息从集群中获取对应的Leader副本 + Replica leader = clusterModel.partition(followerReplica.topicPartition()); + //如果Leader副本所在Broker的Topic Leader数量超过最小Leader分布则进行Leader切换 + if (leader.broker().numLeadersFor(topicName) > minTopicLeadersPerBroker(topicName)) { + if (maybeApplyBalancingAction(clusterModel, leader, Collections.singleton(broker), + LEADERSHIP_MOVEMENT, optimizedGoals, optimizationOptions) != null) { + topicLeaderCount++; + //Topic在当前Broker的Leader分布大于等于最小Leader分布则结束均衡 + if (topicLeaderCount >= minTopicLeadersPerBroker(topicName)) { + return; + } + } + } + } + //根据Topic获取需要Leader数量大于最小Leader分布待迁移的Broker列表 + PriorityQueue brokersWithExcessiveLeaderToMove = getBrokersWithExcessiveLeaderToMove(topicName, clusterModel); + while (!brokersWithExcessiveLeaderToMove.isEmpty()) { + Broker brokerWithExcessiveLeaderToMove = brokersWithExcessiveLeaderToMove.poll(); + List leadersOfTopic = brokerWithExcessiveLeaderToMove.leaderReplicas().stream() + .filter(i -> i.topicPartition().topic().equals(topicName)).collect(Collectors.toList()); + boolean leaderMoved = false; + int leaderMoveCount = leadersOfTopic.size(); + for (Replica leaderOfTopic : leadersOfTopic) { + Broker destinationBroker = maybeApplyBalancingAction(clusterModel, leaderOfTopic, Collections.singleton(broker), + REPLICA_MOVEMENT, optimizedGoals, optimizationOptions); + if (destinationBroker != null) { + leaderMoved = true; + break; + } + } + if (leaderMoved) { + //当前Topic Leader数量在满足最小Leader分布后则结束均衡 + topicLeaderCount++; + if (topicLeaderCount >= minTopicLeadersPerBroker(topicName)) { + return; + } + //分布过多的Broker在进行副本迁移后Topic Leader依然大于最小Leader分布则继续迁移 + leaderMoveCount--; + if (leaderMoveCount > minTopicLeadersPerBroker(topicName)) { + brokersWithExcessiveLeaderToMove.add(brokerWithExcessiveLeaderToMove); + } + } + } + } + + /** + * 根据指定的TopicName,筛选出集群内超过该TopicName Leader平均分布数量的所有Broker并且降序排列 + */ + private PriorityQueue getBrokersWithExcessiveLeaderToMove(String topicName, ClusterModel clusterModel) { + PriorityQueue brokersWithExcessiveLeaderToMove = new PriorityQueue<>((broker1, broker2) -> { + int broker1LeaderCount = broker1.numLeadersFor(topicName); + int broker2LeaderCount = broker2.numLeadersFor(topicName); + int leaderCountCompareResult = Integer.compare(broker2LeaderCount, broker1LeaderCount); + return leaderCountCompareResult == 0 ? Integer.compare(broker1.id(), broker2.id()) : leaderCountCompareResult; + }); + clusterModel.brokers().stream().filter(broker -> broker.numLeadersFor(topicName) > minTopicLeadersPerBroker(topicName)) + .forEach(brokersWithExcessiveLeaderToMove::add); + return brokersWithExcessiveLeaderToMove; + } + + /** + * 下线副本优先处理迁移 + */ + private void moveAwayOfflineReplicas(Broker srcBroker, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + if (srcBroker.currentOfflineReplicas().isEmpty()) { + return; + } + SortedSet eligibleBrokersToMoveOfflineReplicasTo = new TreeSet<>( + Comparator.comparingInt((Broker broker) -> broker.replicas().size()).thenComparingInt(Broker::id)); + Set offlineReplicas = new HashSet<>(srcBroker.currentOfflineReplicas()); + for (Replica offlineReplica : offlineReplicas) { + if (maybeApplyBalancingAction(clusterModel, offlineReplica, eligibleBrokersToMoveOfflineReplicasTo, + REPLICA_MOVEMENT, optimizedGoals, optimizationOptions) == null) { + logger.error(String.format("[%s] offline replica %s from broker %d (has %d replicas) move error", name(), + offlineReplica, srcBroker.id(), srcBroker.replicas().size())); + } + } + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicReplicaDistributionGoal.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicReplicaDistributionGoal.java new file mode 100644 index 00000000..64ef028f --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/optimizer/goals/TopicReplicaDistributionGoal.java @@ -0,0 +1,287 @@ +package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals; + +import com.xiaojukeji.know.streaming.km.rebalance.model.Broker; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Replica; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions; +import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils; + +import java.util.*; +import java.util.stream.Collectors; + +public class TopicReplicaDistributionGoal extends AbstractGoal { + private final Map _balanceUpperLimitByTopic; + private final Map _balanceLowerLimitByTopic; + private Set _brokersAllowedReplicaMove; + private final Map _avgTopicReplicasOnBroker; + + public TopicReplicaDistributionGoal() { + _balanceUpperLimitByTopic = new HashMap<>(); + _balanceLowerLimitByTopic = new HashMap<>(); + _avgTopicReplicasOnBroker = new HashMap<>(); + } + + @Override + protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set optimizedGoals, OptimizationOptions optimizationOptions) { + for (String topic : broker.topics()) { + if (isTopicExcludedFromRebalance(topic)) { + continue; + } + Collection replicas = broker.replicasOfTopicInBroker(topic); + int numTopicReplicas = replicas.size(); + boolean isExcludedForReplicaMove = isExcludedForReplicaMove(broker); + int numOfflineTopicReplicas = GoalUtils.retainCurrentOfflineBrokerReplicas(broker, replicas).size(); + boolean requireLessReplicas = numOfflineTopicReplicas > 0 || numTopicReplicas > _balanceUpperLimitByTopic.get(topic) && !isExcludedForReplicaMove; + boolean requireMoreReplicas = !isExcludedForReplicaMove && numTopicReplicas - numOfflineTopicReplicas < _balanceLowerLimitByTopic.get(topic); + + if (requireLessReplicas) { + rebalanceByMovingReplicasOut(broker, topic, clusterModel, optimizedGoals, optimizationOptions); + } + if (requireMoreReplicas) { + rebalanceByMovingReplicasIn(broker, topic, clusterModel, optimizedGoals, optimizationOptions); + } + } + } + + /** + * 初始化均衡条件: + * 1.Topic平均分布副本数量 + * 2.Topic在平均副本的基础上向上浮动数量 + * 3.Topic在平均副本的基础上向下浮动数量 + */ + @Override + protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) { + Set excludedTopics = optimizationOptions.excludedTopics(); + Set excludedBrokers = optimizationOptions.offlineBrokers(); + Set topicsAllowedRebalance = GoalUtils.getNotExcludeTopics(clusterModel, excludedTopics); + _brokersAllowedReplicaMove = GoalUtils.getNotExcludeBrokers(clusterModel, excludedBrokers); + if (_brokersAllowedReplicaMove.isEmpty()) { + return; + } + for (String topic : topicsAllowedRebalance) { + int numTopicReplicas = clusterModel.numTopicReplicas(topic); + _avgTopicReplicasOnBroker.put(topic, numTopicReplicas / (double) _brokersAllowedReplicaMove.size()); + _balanceUpperLimitByTopic.put(topic, balanceUpperLimit(topic, optimizationOptions)); + _balanceLowerLimitByTopic.put(topic, balanceLowerLimit(topic, optimizationOptions)); + } + } + + /** + * 指定Topic平均副本向下浮动,默认10% + */ + private Integer balanceLowerLimit(String topic, OptimizationOptions optimizationOptions) { + return (int) Math.floor(_avgTopicReplicasOnBroker.get(topic) + * Math.max(0, (1 - optimizationOptions.topicReplicaThreshold()))); + } + + /** + * 指定Topic平均副本向上浮动,默认10% + */ + private Integer balanceUpperLimit(String topic, OptimizationOptions optimizationOptions) { + return (int) Math.ceil(_avgTopicReplicasOnBroker.get(topic) + * (1 + optimizationOptions.topicReplicaThreshold())); + } + + @Override + protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) { + Broker sourceBroker = clusterModel.broker(action.sourceBrokerId()); + if (sourceBroker.replica(action.topicPartition()).isCurrentOffline()) { + return action.balancingAction() == ActionType.REPLICA_MOVEMENT; + } + Broker destinationBroker = clusterModel.broker(action.destinationBrokerId()); + String sourceTopic = action.topic(); + return isReplicaCountAddUpperLimit(sourceTopic, destinationBroker) + && (isExcludedForReplicaMove(sourceBroker) || isReplicaCountRemoveLowerLimit(sourceTopic, sourceBroker)); + + } + + @Override + public String name() { + return TopicReplicaDistributionGoal.class.getSimpleName(); + } + + @Override + public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) { + Broker sourceBroker = clusterModel.broker(action.sourceBrokerId()); + Broker destinationBroker = clusterModel.broker(action.destinationBrokerId()); + String sourceTopic = action.topic(); + switch (action.balancingAction()) { + case LEADERSHIP_MOVEMENT: + return ActionAcceptance.ACCEPT; + case REPLICA_MOVEMENT: + return (isReplicaCountAddUpperLimit(sourceTopic, destinationBroker) + && (isExcludedForReplicaMove(sourceBroker) + || isReplicaCountRemoveLowerLimit(sourceTopic, sourceBroker))) ? ActionAcceptance.ACCEPT : ActionAcceptance.REJECT; + default: + throw new IllegalArgumentException("Unsupported balancing action " + action.balancingAction() + " is provided."); + } + } + + /** + * 指定的Broker上存在Topic副本数大于阈值则迁出副本 + */ + private boolean rebalanceByMovingReplicasOut(Broker broker, + String topic, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + //筛选出现低于UpperLimit的所有Broker做为存放目标 + SortedSet candidateBrokers = new TreeSet<>( + Comparator.comparingInt((Broker b) -> b.numReplicasOfTopicInBroker(topic)).thenComparingInt(Broker::id)); + Set filterUpperLimitBroker = clusterModel.brokers().stream().filter(b -> b.numReplicasOfTopicInBroker(topic) < _balanceUpperLimitByTopic.get(topic)).collect(Collectors.toSet()); + candidateBrokers.addAll(filterUpperLimitBroker); + Collection replicasOfTopicInBroker = broker.replicasOfTopicInBroker(topic); + int numReplicasOfTopicInBroker = replicasOfTopicInBroker.size(); + int numOfflineTopicReplicas = GoalUtils.retainCurrentOfflineBrokerReplicas(broker, replicasOfTopicInBroker).size(); + int balanceUpperLimitForSourceBroker = isExcludedForReplicaMove(broker) ? 0 : _balanceUpperLimitByTopic.get(topic); + boolean wasUnableToMoveOfflineReplica = false; + + for (Replica replica : replicasToMoveOut(broker, topic)) { + //当前Broker没有离线副本及Topic的副本数量低于UpperLimit则结束均衡 + if (wasUnableToMoveOfflineReplica && !replica.isCurrentOffline() && numReplicasOfTopicInBroker <= balanceUpperLimitForSourceBroker) { + return false; + } + boolean wasOffline = replica.isCurrentOffline(); + Broker b = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, ActionType.REPLICA_MOVEMENT, + optimizedGoals, optimizationOptions); + // Only check if we successfully moved something. + if (b != null) { + if (wasOffline) { + numOfflineTopicReplicas--; + } + if (--numReplicasOfTopicInBroker <= (numOfflineTopicReplicas == 0 ? balanceUpperLimitForSourceBroker : 0)) { + return false; + } + + // Remove and reinsert the broker so the order is correct. + candidateBrokers.remove(b); + if (b.numReplicasOfTopicInBroker(topic) < _balanceUpperLimitByTopic.get(topic)) { + candidateBrokers.add(b); + } + } else if (wasOffline) { + wasUnableToMoveOfflineReplica = true; + } + } + return !broker.replicasOfTopicInBroker(topic).isEmpty(); + } + + /** + * 1.离线副本优行处理 + * 2.小分区号优行处理 + */ + private SortedSet replicasToMoveOut(Broker broker, String topic) { + SortedSet replicasToMoveOut = new TreeSet<>((r1, r2) -> { + boolean r1Offline = broker.currentOfflineReplicas().contains(r1); + boolean r2Offline = broker.currentOfflineReplicas().contains(r2); + if (r1Offline && !r2Offline) { + return -1; + } else if (!r1Offline && r2Offline) { + return 1; + } + if (r1.topicPartition().partition() > r2.topicPartition().partition()) { + return 1; + } else if (r1.topicPartition().partition() < r2.topicPartition().partition()) { + return -1; + } + return 0; + }); + replicasToMoveOut.addAll(broker.replicasOfTopicInBroker(topic)); + return replicasToMoveOut; + } + + /** + * Topic副本数>最低阈值的副本，迁到指定的Broker上 + */ + private boolean rebalanceByMovingReplicasIn(Broker broker, + String topic, + ClusterModel clusterModel, + Set optimizedGoals, + OptimizationOptions optimizationOptions) { + PriorityQueue eligibleBrokers = new PriorityQueue<>((b1, b2) -> { + Collection replicasOfTopicInB2 = b2.replicasOfTopicInBroker(topic); + int numReplicasOfTopicInB2 = replicasOfTopicInB2.size(); + int numOfflineTopicReplicasInB2 = GoalUtils.retainCurrentOfflineBrokerReplicas(b2, replicasOfTopicInB2).size(); + Collection replicasOfTopicInB1 = b1.replicasOfTopicInBroker(topic); + int numReplicasOfTopicInB1 = replicasOfTopicInB1.size(); + int numOfflineTopicReplicasInB1 = GoalUtils.retainCurrentOfflineBrokerReplicas(b1, replicasOfTopicInB1).size(); + + int resultByOfflineReplicas = Integer.compare(numOfflineTopicReplicasInB2, numOfflineTopicReplicasInB1); + if (resultByOfflineReplicas == 0) { + int resultByAllReplicas = Integer.compare(numReplicasOfTopicInB2, numReplicasOfTopicInB1); + return resultByAllReplicas == 0 ? Integer.compare(b1.id(), b2.id()) : resultByAllReplicas; + } + return resultByOfflineReplicas; + }); + //筛选当前Topic高于LowerLimit、存在离线副本、的所有Broker做为需要迁的副本 + for (Broker sourceBroker : clusterModel.brokers()) { + if (sourceBroker.numReplicasOfTopicInBroker(topic) > _balanceLowerLimitByTopic.get(topic) + || !sourceBroker.currentOfflineReplicas().isEmpty() || isExcludedForReplicaMove(sourceBroker)) { + eligibleBrokers.add(sourceBroker); + } + } + Collection replicasOfTopicInBroker = broker.replicasOfTopicInBroker(topic); + int numReplicasOfTopicInBroker = replicasOfTopicInBroker.size(); + //当前Broker做为存放目标 + Set candidateBrokers = Collections.singleton(broker); + while (!eligibleBrokers.isEmpty()) { + Broker sourceBroker = eligibleBrokers.poll(); + SortedSet replicasToMove = replicasToMoveOut(sourceBroker, topic); + int numOfflineTopicReplicas = GoalUtils.retainCurrentOfflineBrokerReplicas(sourceBroker, replicasToMove).size(); + + for (Replica replica : replicasToMove) { + boolean wasOffline = replica.isCurrentOffline(); + Broker b = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, ActionType.REPLICA_MOVEMENT, + optimizedGoals, optimizationOptions); + if (b != null) { + if (wasOffline) { + numOfflineTopicReplicas--; + } + if (++numReplicasOfTopicInBroker >= _balanceLowerLimitByTopic.get(topic)) { + return false; + } + if (!eligibleBrokers.isEmpty() && numOfflineTopicReplicas == 0 + && sourceBroker.numReplicasOfTopicInBroker(topic) < eligibleBrokers.peek().numReplicasOfTopicInBroker(topic)) { + eligibleBrokers.add(sourceBroker); + break; + } + } + } + } + return true; + } + + /** + * 目标Broker增加副本后，Topic副本数<=最高阈值 + */ + private boolean isReplicaCountAddUpperLimit(String topic, Broker destinationBroker) { + int numTopicReplicas = destinationBroker.numReplicasOfTopicInBroker(topic); + int brokerBalanceUpperLimit = _balanceUpperLimitByTopic.get(topic); + return numTopicReplicas + 1 <= brokerBalanceUpperLimit; + } + + /** + * 源Broker迁走副本后，Topic副本数>=最低阈值 + */ + private boolean isReplicaCountRemoveLowerLimit(String topic, Broker sourceBroker) { + int numTopicReplicas = sourceBroker.numReplicasOfTopicInBroker(topic); + int brokerBalanceLowerLimit = _balanceLowerLimitByTopic.get(topic); + return numTopicReplicas - 1 >= brokerBalanceLowerLimit; + } + + /** + * 判断指定的Broker是否可以进行副本迁移操作 + */ + private boolean isExcludedForReplicaMove(Broker broker) { + return !_brokersAllowedReplicaMove.contains(broker); + } + + /** + * 判断指定的Topic是否在可均衡的列表中 + */ + private boolean isTopicExcludedFromRebalance(String topic) { + return _avgTopicReplicasOnBroker.get(topic) == null; + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/CommandLineUtils.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/CommandLineUtils.java new file mode 100644 index 00000000..54412665 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/CommandLineUtils.java @@ -0,0 +1,21 @@ +package com.xiaojukeji.know.streaming.km.rebalance.utils; + +import joptsimple.OptionParser; + +import java.io.IOException; + +public class CommandLineUtils { + + /** + * Print usage and exit + */ + public static void printUsageAndDie(OptionParser parser, String message) { + try { + System.err.println(message); + parser.printHelpOn(System.err); + System.exit(1); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/GoalUtils.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/GoalUtils.java new file mode 100644 index 00000000..bd6358a1 --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/GoalUtils.java @@ -0,0 +1,59 @@ +package com.xiaojukeji.know.streaming.km.rebalance.utils; + +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceGoal; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceThreshold; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.HostEnv; +import com.xiaojukeji.know.streaming.km.rebalance.model.*; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.AnalyzerUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +public class GoalUtils { + private static final Logger logger = LoggerFactory.getLogger(GoalUtils.class); + + public static Set getNotExcludeTopics(ClusterModel clusterModel, Set excludedTopics) { + return clusterModel.topics().stream().filter(topicName -> !excludedTopics.contains(topicName)).collect(Collectors.toSet()); + } + + public static Set getNotExcludeBrokers(ClusterModel clusterModel, Set excludedBrokers) { + return clusterModel.brokers().stream().filter(broker -> !excludedBrokers.contains(broker.id())).collect(Collectors.toSet()); + } + + /** + * 在Broker上获取指定的离线副本列表 + */ + public static Set retainCurrentOfflineBrokerReplicas(Broker broker, Collection replicas) { + Set offlineReplicas = new HashSet<>(replicas); + offlineReplicas.retainAll(broker.currentOfflineReplicas()); + return offlineReplicas; + } + + public static ClusterModel getInitClusterModel(BalanceParameter parameter) { + logger.info("Cluster model initialization"); + List hostsEnv = parameter.getHardwareEnv(); + Map capacities = new HashMap<>(); + for (HostEnv env : hostsEnv) { + Capacity capacity = new Capacity(); + capacity.setCapacity(Resource.CPU, env.getCpu()); + capacity.setCapacity(Resource.DISK, env.getDisk()); + capacity.setCapacity(Resource.NW_IN, env.getNetwork()); + capacity.setCapacity(Resource.NW_OUT, env.getNetwork()); + capacities.put(env.getId(), capacity); + } + return Supplier.load(parameter.getCluster(), parameter.getBeforeSeconds(), parameter.getKafkaConfig(), + parameter.getEsRestURL(), parameter.getEsIndexPrefix(), capacities, AnalyzerUtils.getSplitTopics(parameter.getIgnoredTopics())); + } + + public static Map getBalanceThreshold(BalanceParameter parameter, double[] clusterAvgResource) { + Map balanceThreshold = new HashMap<>(); + balanceThreshold.put(BalanceGoal.DISK.goal(), new BalanceThreshold(Resource.DISK, parameter.getDiskThreshold(), clusterAvgResource[Resource.DISK.id()])); + balanceThreshold.put(BalanceGoal.NW_IN.goal(), new BalanceThreshold(Resource.NW_IN, parameter.getNetworkInThreshold(), clusterAvgResource[Resource.NW_IN.id()])); + balanceThreshold.put(BalanceGoal.NW_OUT.goal(), new BalanceThreshold(Resource.NW_OUT, parameter.getNetworkOutThreshold(), clusterAvgResource[Resource.NW_OUT.id()])); + return balanceThreshold; + } + +} diff --git a/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/MetadataUtils.java b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/MetadataUtils.java new file mode 100644 index 00000000..dd83659f --- /dev/null +++ b/km-extends/km-rebalance/src/main/java/com/xiaojukeji/know/streaming/km/rebalance/utils/MetadataUtils.java @@ -0,0 +1,92 @@ +package com.xiaojukeji.know.streaming.km.rebalance.utils; + +import org.apache.kafka.clients.*; +import org.apache.kafka.clients.consumer.internals.NoAvailableBrokersException; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.network.ChannelBuilder; +import org.apache.kafka.common.network.NetworkReceive; +import org.apache.kafka.common.network.Selector; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +/** + * @author leewei + * @date 2022/5/27 + */ +public class MetadataUtils { + private static final Logger logger = LoggerFactory.getLogger(MetadataUtils.class); + + public static Cluster metadata(Properties props) { + props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.BytesSerializer"); + props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.BytesSerializer"); + ProducerConfig config = new ProducerConfig(props); + + Time time = Time.SYSTEM; + LogContext logContext = new LogContext("Metadata client"); + + ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time, logContext); + Selector selector = new Selector( + NetworkReceive.UNLIMITED, + config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), + new org.apache.kafka.common.metrics.Metrics(), + time, + "metadata-client", + Collections.singletonMap("client", "metadata-client"), + false, + channelBuilder, + logContext + ); + + NetworkClient networkClient = new NetworkClient( + selector, + new ManualMetadataUpdater(), + "metadata-client", + 1, + config.getLong(ProducerConfig.RECONNECT_BACKOFF_MS_CONFIG), + config.getLong(ProducerConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG), + config.getInt(ProducerConfig.SEND_BUFFER_CONFIG), + config.getInt(ProducerConfig.RECEIVE_BUFFER_CONFIG), + config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG), + config.getLong(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG), + config.getLong(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG), + ClientDnsLookup.DEFAULT, + time, + true, + new ApiVersions(), + logContext + ); + + try { + List nodes = config.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); + for (int i = 0; i < nodes.size(); i++) { + Node sourceNode = new Node(i, Utils.getHost(nodes.get(i)), Utils.getPort(nodes.get(i))); + try { + if (NetworkClientUtils.awaitReady(networkClient, sourceNode, time, 10 * 1000)) { + ClientRequest clientRequest = networkClient.newClientRequest(String.valueOf(i), MetadataRequest.Builder.allTopics(), + time.milliseconds(), true); + ClientResponse clientResponse = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, time); + MetadataResponse metadataResponse = (MetadataResponse) clientResponse.responseBody(); + return metadataResponse.buildCluster(); + } + } catch (IOException e) { + logger.warn("Connection to " + sourceNode + " error", e); + } + } + throw new NoAvailableBrokersException(); + } finally { + networkClient.close(); + } + } +} diff --git a/km-extends/km-rebalance/src/main/resources/META-INF/services/com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal b/km-extends/km-rebalance/src/main/resources/META-INF/services/com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal new file mode 100644 index 00000000..4697e641 --- /dev/null +++ b/km-extends/km-rebalance/src/main/resources/META-INF/services/com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal @@ -0,0 +1,5 @@ +com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.TopicLeadersDistributionGoal +com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.TopicReplicaDistributionGoal +com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.DiskDistributionGoal +com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.NetworkInboundDistributionGoal +com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.NetworkOutboundDistributionGoal \ No newline at end of file diff --git a/km-extends/km-rebalance/src/main/resources/MetricsQuery.json b/km-extends/km-rebalance/src/main/resources/MetricsQuery.json new file mode 100644 index 00000000..955a2329 --- /dev/null +++ b/km-extends/km-rebalance/src/main/resources/MetricsQuery.json @@ -0,0 +1,75 @@ +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { + "term": { + "clusterPhyId": "" + } + }, + { + "range": { + "timestamp": { + "gte": "now-s", + "lte": "now" + } + } + } + ] + } + }, + "aggs": { + "by_topic": { + "terms": { + "field": "topic", + "size": 5000 + }, + "aggs": { + "by_partition": { + "terms": { + "field": "partitionId", + "size": 2000 + }, + "aggs": { + "avg_cpu": { + "avg": { + "field": "metrics.CPU" + } + }, + "avg_bytes_in": { + "avg": { + "field": "metrics.BytesIn" + } + }, + "avg_bytes_out": { + "avg": { + "field": "metrics.BytesOut" + } + }, + "avg_disk": { + "avg": { + "field": "metrics.LogSize" + } + }, + "lastest_disk": { + "top_hits": { + "sort": [ + { + "timestamp": { + "order": "desc" + } + } + ], + "_source": { + "includes": [ "metrics.LogSize"] + }, + "size": 1 + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/km-extends/km-rebalance/src/test/java/com/xiaojukeji/know/streaming/km/rebalance/Test.java b/km-extends/km-rebalance/src/test/java/com/xiaojukeji/know/streaming/km/rebalance/Test.java new file mode 100644 index 00000000..c10deb8c --- /dev/null +++ b/km-extends/km-rebalance/src/test/java/com/xiaojukeji/know/streaming/km/rebalance/Test.java @@ -0,0 +1,93 @@ +package com.xiaojukeji.know.streaming.km.rebalance; + +import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.HostEnv; +import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel; +import com.xiaojukeji.know.streaming.km.rebalance.model.Partition; +import com.xiaojukeji.know.streaming.km.rebalance.model.Replica; +import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult; +import com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal; +import org.apache.kafka.clients.CommonClientConfigs; + +import java.util.*; + +/** + * @author leewei + * @date 2022/5/30 + */ +public class Test { + private static HostEnv getHostEnv(int id) { + HostEnv env = new HostEnv(); + env.setId(id); + env.setCpu(3200); // 32C + env.setDisk(2 * 1024D * 1024 * 1024 * 1024); // 2T + env.setNetwork(10 * 1024D * 1024 * 1024); // 10G + return env; + } + + public static void main(String[] args) { + ServiceLoader loader = ServiceLoader.load(Goal.class); + for (Goal goal : loader) { + System.out.println(goal.name() + " ： " + goal); + } + + BalanceParameter balanceParameter = new BalanceParameter(); + + balanceParameter.setDiskThreshold(0.05); + balanceParameter.setCpuThreshold(0.05); + balanceParameter.setNetworkInThreshold(0.05); + balanceParameter.setNetworkOutThreshold(0.05); + + List envList = new ArrayList<>(); + envList.add(getHostEnv(6416)); + envList.add(getHostEnv(6417)); + envList.add(getHostEnv(6431)); + envList.add(getHostEnv(6432)); + envList.add(getHostEnv(6553)); + balanceParameter.setHardwareEnv(envList); + + // String goals = "DiskDistributionGoal,NetworkInboundDistributionGoal,NetworkOutboundDistributionGoal"; + // String goals = "DiskDistributionGoal"; + String goals = "NetworkOutboundDistributionGoal"; + + balanceParameter.setGoals(Arrays.asList(goals.split(","))); + Properties kafkaConfig = new Properties(); + kafkaConfig.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "10.96.64.16:7262"); + // kafkaConfig.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "10.96.64.16:9162,10.96.64.17:9162,10.96.64.31:9162"); + balanceParameter.setKafkaConfig(kafkaConfig); + balanceParameter.setEsRestURL("10.96.64.13:8061"); + balanceParameter.setEsIndexPrefix("ks_kafka_partition_metric_"); + + balanceParameter.setBeforeSeconds(300); + + balanceParameter.setCluster("293"); + + ExecutionRebalance exec = new ExecutionRebalance(); + OptimizerResult optimizerResult = exec.optimizations(balanceParameter); + + System.out.println(optimizerResult.resultJsonOverview()); + System.out.println(optimizerResult.resultJsonDetailed()); + System.out.println(optimizerResult.resultJsonTask()); + System.out.println(optimizerResult.resultJsonBalanceActionHistory()); + + ClusterModel model = optimizerResult.clusterModel(); + + System.out.println("---moved partitions----"); + // moved partitions + for (String topic : model.topics()) { + model.topic(topic).stream() + .filter(Partition::isChanged) + .forEach(partition -> { + System.out.println("---> " + partition); + System.out.println(partition.topicPartition() + " leader change: " + partition.isLeaderChanged() + " " + (partition.isLeaderChanged() ? partition.originalLeaderBroker().id() + " -> " + partition.leader().broker().id() : "")); + partition.replicas().stream() + .filter(Replica::isChanged).forEach(r -> { + //System.out.println(r); + System.out.println(partition.topicPartition() + " replica moved: " + r.original().broker().id() + "（" + r.original().isLeader() + "） -> " + r.broker().id() + " (" + r.isLeader() + ")"); + }); + }); + } + System.out.println("---end moved partitions----"); + } +} diff --git a/km-extends/km-rebalance/src/test/resources/log4j.properties b/km-extends/km-rebalance/src/test/resources/log4j.properties new file mode 100644 index 00000000..4f185dd9 --- /dev/null +++ b/km-extends/km-rebalance/src/test/resources/log4j.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=INFO, stdout + +# Console output... +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobConfigDao.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobConfigDao.java new file mode 100644 index 00000000..526e49e3 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobConfigDao.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobConfigPO; +import org.springframework.stereotype.Repository; + +/** + * 集群均衡任务 Dao + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Repository +@EnterpriseLoadReBalance +public interface ClusterBalanceJobConfigDao extends BaseMapper { + +} diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobDao.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobDao.java new file mode 100644 index 00000000..278d99ba --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceJobDao.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceJobPO; +import org.springframework.stereotype.Repository; + +/** + * 集群均衡任务 Dao + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Repository +@EnterpriseLoadReBalance +public interface ClusterBalanceJobDao extends BaseMapper { + + void addClusterBalanceJob(ClusterBalanceJobPO clusterBalanceJobPO); + +} diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceReassignDao.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceReassignDao.java new file mode 100644 index 00000000..141e8ca0 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/mysql/enterprise/rebalance/ClusterBalanceReassignDao.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2015, WINIT and/or its affiliates. All rights reserved. Use, Copy is subject to authorized license. + */ +package com.xiaojukeji.know.streaming.km.persistence.mysql.enterprise.rebalance; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.po.ClusterBalanceReassignPO; +import org.springframework.stereotype.Repository; + +import java.util.List; + +/** + * 集群平衡迁移详情 Dao + * + * @author fengqiongfeng + * @date 2022-05-23 + */ +@Repository +@EnterpriseLoadReBalance +public interface ClusterBalanceReassignDao extends BaseMapper { + + int addBatch(List reassignPOList); +} diff --git a/km-persistence/src/main/resources/mybatis/ClusterBalanceJobMapper.xml b/km-persistence/src/main/resources/mybatis/ClusterBalanceJobMapper.xml new file mode 100644 index 00000000..cc6d1b7a --- /dev/null +++ b/km-persistence/src/main/resources/mybatis/ClusterBalanceJobMapper.xml @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/km-persistence/src/main/resources/mybatis/ClusterBalanceReassignMapper.xml b/km-persistence/src/main/resources/mybatis/ClusterBalanceReassignMapper.xml new file mode 100644 index 00000000..a75cb329 --- /dev/null +++ b/km-persistence/src/main/resources/mybatis/ClusterBalanceReassignMapper.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + (#{item.jobId},#{item.clusterId},#{item.topicName},#{item.partitionId},#{item.originalBrokerIds},#{item.reassignBrokerIds},#{item.startTime},#{item.finishedTime},#{item.extendData},#{item.status},#{item.createTime},#{item.updateTime}) + + + + + + #{item} + + + diff --git a/km-rest/pom.xml b/km-rest/pom.xml index 8814a8a5..0dfc429e 100644 --- a/km-rest/pom.xml +++ b/km-rest/pom.xml @@ -22,6 +22,11 @@ + + com.xiaojukeji.kafka + km-testing + ${project.parent.version} + com.xiaojukeji.kafka km-account @@ -32,6 +37,11 @@ km-monitor ${project.parent.version} + + com.xiaojukeji.kafka + km-license + ${project.parent.version} + com.xiaojukeji.kafka km-core diff --git a/km-rest/src/main/java/com/xiaojukeji/know/streaming/km/rest/api/v3/enterprise/rebalance/ClusterBalanceController.java b/km-rest/src/main/java/com/xiaojukeji/know/streaming/km/rest/api/v3/enterprise/rebalance/ClusterBalanceController.java new file mode 100644 index 00000000..5731c5a3 --- /dev/null +++ b/km-rest/src/main/java/com/xiaojukeji/know/streaming/km/rest/api/v3/enterprise/rebalance/ClusterBalanceController.java @@ -0,0 +1,88 @@ +package com.xiaojukeji.know.streaming.km.rest.api.v3.enterprise.rebalance; + + +import com.didiglobal.logi.security.util.HttpRequestUtil; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceOverviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalancePreviewDTO; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.dto.ClusterBalanceStrategyDTO; +import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; +import com.xiaojukeji.know.streaming.km.common.enterprise.rebalance.bean.vo.*; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +@EnterpriseLoadReBalance +@Api(tags = Constant.SWAGGER_API_TAG_PREFIX + "集群均衡-相关接口(REST)") +@RestController +@RequestMapping(ApiPrefix.API_V3_PREFIX) +public class ClusterBalanceController { + + @Autowired + private ClusterBalanceService clusterBalanceService; + + @ApiOperation(value = "集群均衡配置信息") + @GetMapping(value = "clusters/{clusterPhyId}/balance-config") + @ResponseBody + public Result balanceConfig(@PathVariable Long clusterPhyId) { + return clusterBalanceService.config(clusterPhyId); + } + + @ApiOperation(value = "集群周期均衡信息") + @GetMapping(value = "clusters/{clusterPhyId}/balance-state") + @ResponseBody + public Result state(@PathVariable Long clusterPhyId) { + return clusterBalanceService.state(clusterPhyId); + } + + @ApiOperation(value = "集群均衡列表信息") + @PostMapping(value = "clusters/{clusterPhyId}/balance-overview") + @ResponseBody + public PaginationResult overview(@PathVariable Long clusterPhyId, + @RequestBody ClusterBalanceOverviewDTO dto) { + return clusterBalanceService.overview(clusterPhyId, dto); + } + + @ApiOperation(value = "集群均衡历史信息") + @PostMapping(value = "clusters/{clusterPhyId}/balance-history") + @ResponseBody + public PaginationResult history(@PathVariable Long clusterPhyId, + @RequestBody PaginationBaseDTO dto) { + return clusterBalanceService.history(clusterPhyId, dto); + } + + @ApiOperation(value = "集群均衡计划信息") + @GetMapping(value = "clusters/{clusterPhyId}/balance-plan/{jobId}") + @ResponseBody + public Result plan(@PathVariable Long clusterPhyId, @PathVariable Long jobId) { + return clusterBalanceService.plan(clusterPhyId, jobId); + } + + @ApiOperation(value = "集群均衡状态信息") + @GetMapping(value = "clusters/{clusterPhyId}/balance-schedule/{jobId}") + @ResponseBody + public Result schedule(@PathVariable Long clusterPhyId, @PathVariable Long jobId) { + return clusterBalanceService.schedule(clusterPhyId, jobId); + } + + @ApiOperation(value = "集群立即均衡预览信息") + @PostMapping(value = "clusters/{clusterPhyId}/balance-preview") + @ResponseBody + public Result preview(@PathVariable Long clusterPhyId, + @RequestBody ClusterBalancePreviewDTO clusterBalancePreviewDTO) { + return clusterBalanceService.preview(clusterPhyId, clusterBalancePreviewDTO); + } + + @ApiOperation(value = "集群均衡") + @PostMapping(value = "clusters/{clusterPhyId}/balance-strategy") + @ResponseBody + public Result strategy(@PathVariable Long clusterPhyId, @RequestBody ClusterBalanceStrategyDTO dto) { + return clusterBalanceService.strategy(clusterPhyId, dto, HttpRequestUtil.getOperator()); + } +} diff --git a/km-rest/src/main/resources/application.yml b/km-rest/src/main/resources/application.yml index 4a4b7f1c..d857dcc4 100644 --- a/km-rest/src/main/resources/application.yml +++ b/km-rest/src/main/resources/application.yml @@ -89,6 +89,13 @@ es: io-thread-cnt: 2 max-retry-cnt: 5 + +# 集群自动均衡相关配置 +cluster-balance: + ignored-topics: + time-second: 300 + + # 普罗米修斯指标导出相关配置 management: endpoints: @@ -105,4 +112,11 @@ management: descriptions: true enabled: true tags: - application: know-streaming \ No newline at end of file + application: know-streaming + +# license相关配置 +license: + app-name: know-streaming + server: http://127.0.0.1:8778 + signature: 1234567890 + token: license-builtin-token \ No newline at end of file diff --git a/km-task/src/main/java/com/xiaojukeji/know/streaming/km/task/enterprise/rebalance/job/ClusterBalanceJobTask.java b/km-task/src/main/java/com/xiaojukeji/know/streaming/km/task/enterprise/rebalance/job/ClusterBalanceJobTask.java new file mode 100644 index 00000000..c29d8306 --- /dev/null +++ b/km-task/src/main/java/com/xiaojukeji/know/streaming/km/task/enterprise/rebalance/job/ClusterBalanceJobTask.java @@ -0,0 +1,54 @@ +package com.xiaojukeji.know.streaming.km.task.enterprise.rebalance.job; + +import com.didiglobal.logi.job.annotation.Task; +import com.didiglobal.logi.job.common.TaskResult; +import com.didiglobal.logi.job.core.consensual.ConsensualEnum; +import com.xiaojukeji.know.streaming.km.common.annotations.enterprise.EnterpriseLoadReBalance; +import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; +import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceJobService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceReassignService; +import com.xiaojukeji.know.streaming.km.core.enterprise.rebalance.service.ClusterBalanceService; +import com.xiaojukeji.know.streaming.km.task.AbstractAsyncCommonDispatchTask; +import org.springframework.beans.factory.annotation.Autowired; + +@EnterpriseLoadReBalance +@Task(name = "clusterBalanceJobTask", + description = "周期性校验是否到点能生成集群均衡任务，以及更新任务状态", + cron = "0 0/1 * * * ? *", + autoRegister = true, + consensual = ConsensualEnum.BROADCAST, + timeout = 2 * 60) +public class ClusterBalanceJobTask extends AbstractAsyncCommonDispatchTask { + + @Autowired + private ClusterBalanceService clusterBalanceService; + + @Autowired + private ClusterBalanceJobService clusterBalanceJobService; + + @Autowired + private ClusterBalanceReassignService clusterBalanceReassignService; + + @Override + public TaskResult processClusterTask(ClusterPhy clusterPhy, long triggerTimeUnitMs) throws Exception { + clusterBalanceService.createScheduleJob(clusterPhy.getId(), triggerTimeUnitMs); + // 获取迁移中的任务 + Long jobId = clusterBalanceJobService.getOneRunningJob(clusterPhy.getId()); + if (jobId == null) { + // 当前无任务 + return TaskResult.SUCCESS; + } + + // 更新任务的状态 + Result rv = clusterBalanceJobService.verifyClusterBalanceAndUpdateStatue(jobId); + + // 更新同步进度信息 + clusterBalanceReassignService.getAndUpdateSubJobExtendData(jobId); + + //根据策略生成新的或更新迁移任务 + Result r = clusterBalanceJobService.generateReassignmentForStrategy(clusterPhy.getId(), jobId); + + return rv.failed()||r.failed() ? TaskResult.FAIL: TaskResult.SUCCESS; + } +} diff --git a/pom.xml b/pom.xml index a168b74c..3e1d28ae 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ - 3.1.0 + enterprise-3.1.0 8 8 @@ -53,10 +53,13 @@ km-biz km-extends/km-account km-extends/km-monitor + km-extends/km-license + km-extends/km-rebalance km-task km-collector km-rest km-dist + km-enterprise/km-testing