mirror of
https://github.com/didi/KnowStreaming.git
synced 2026-01-02 02:02:13 +08:00
合并3.3.0企业版改动
This commit is contained in:
zengqiao
@@ -1,40 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.xiaojukeji.kafka</groupId>
|
||||
<artifactId>km-license</artifactId>
|
||||
<version>${km.revision}</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<parent>
|
||||
<artifactId>km</artifactId>
|
||||
<groupId>com.xiaojukeji.kafka</groupId>
|
||||
<version>${km.revision}</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.xiaojukeji.kafka</groupId>
|
||||
<artifactId>km-common</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.xiaojukeji.kafka</groupId>
|
||||
<artifactId>km-persistence</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.xiaojukeji.kafka</groupId>
|
||||
<artifactId>km-core</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -1,69 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.license.service.LicenseService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.servlet.HandlerInterceptor;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix.API_V3_PREFIX;
|
||||
|
||||
@Component
|
||||
public class LicenseInterceptor implements HandlerInterceptor {
|
||||
private static final ILog LOGGER = LogFactory.getLog(LicenseInterceptor.class);
|
||||
|
||||
private static final String PHYSICAL_CLUSTER_URL = API_V3_PREFIX + "physical-clusters";
|
||||
private static final String UTF_8 = "utf-8";
|
||||
|
||||
@Autowired
|
||||
private LicenseService licenseService;
|
||||
|
||||
/**
|
||||
* 拦截预处理
|
||||
* @return boolean false:拦截, 不向下执行, true:放行
|
||||
*/
|
||||
@Override
|
||||
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
|
||||
if (PHYSICAL_CLUSTER_URL.equals( request.getRequestURI() ) &&
|
||||
"POST".equals( request.getMethod() )) {
|
||||
|
||||
Result<Void> result = licenseService.addClusterLimit();
|
||||
if (result.failed()) {
|
||||
// 如果出错，构造错误信息
|
||||
OutputStream out = null;
|
||||
try {
|
||||
response.setCharacterEncoding(UTF_8);
|
||||
response.setContentType("text/json");
|
||||
out = response.getOutputStream();
|
||||
out.write(ConvertUtil.obj2Json(result).getBytes(UTF_8));
|
||||
out.flush();
|
||||
} catch (IOException e) {
|
||||
LOGGER.error( "method=preHandle||msg=physical-clusters add exception! ", e);
|
||||
} finally {
|
||||
try {
|
||||
if (out != null) {
|
||||
out.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
LOGGER.error( "method=preHandle||msg=outputStream close exception! ", e);
|
||||
}
|
||||
}
|
||||
|
||||
// 拒绝向下执行
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 未达到限制，继续后续的执行
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
|
||||
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Configuration
|
||||
public class LicenseWebConfig implements WebMvcConfigurer {
|
||||
@Autowired
|
||||
private LicenseInterceptor licenseInterceptor;
|
||||
|
||||
@Override
|
||||
public void addInterceptors(InterceptorRegistry registry) {
|
||||
// 会进行拦截的接口
|
||||
registry.addInterceptor(licenseInterceptor).addPathPatterns(ApiPrefix.API_PREFIX + "**");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.bean;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Data
|
||||
public class KmLicense {
|
||||
private int clusters;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.bean;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Data
|
||||
public class KmLicenseUsageDetail {
|
||||
/**
|
||||
* 上报的 ks 的节点
|
||||
*/
|
||||
private String host;
|
||||
/**
|
||||
* 上报的 ks 的集群的所有的节点
|
||||
*/
|
||||
private List<String> hosts;
|
||||
/**
|
||||
* 上报的 ks 集群中 kafka 集群数量
|
||||
*/
|
||||
private int clusters;
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.bean;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Data
|
||||
public class LicenseInfo<T> {
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private int status;
|
||||
|
||||
/**
|
||||
* license 过期时间，单位秒
|
||||
*/
|
||||
private Long expiredDate;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private String app;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private String type;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private T info;
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.bean;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Data
|
||||
public class LicenseResult<T> {
|
||||
String err;
|
||||
T reply;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.bean;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Data
|
||||
public class LicenseUsage {
|
||||
/**
|
||||
* 上报时间戳
|
||||
*/
|
||||
private Long timeStamp;
|
||||
|
||||
/**
|
||||
* uuid
|
||||
*/
|
||||
private String uuid;
|
||||
|
||||
/**
|
||||
* 业务数据
|
||||
*/
|
||||
private String data;
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.controller;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.ApiPrefix;
|
||||
import com.xiaojukeji.know.streaming.km.license.service.LicenseService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping(ApiPrefix.API_V3_PREFIX)
|
||||
public class LicenseController {
|
||||
|
||||
@Autowired
|
||||
private LicenseService licenseService;
|
||||
|
||||
@GetMapping(value = "license")
|
||||
@ResponseBody
|
||||
public Result<Void> check() {
|
||||
return licenseService.check();
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.service;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
|
||||
public interface LicenseService {
|
||||
|
||||
/**
|
||||
* 是否达到了 license 现在的集群数量
|
||||
* @return
|
||||
*/
|
||||
Result<Void> addClusterLimit();
|
||||
|
||||
/**
|
||||
* 校验 license 是否通过
|
||||
* @return
|
||||
*/
|
||||
Result<Void> check();
|
||||
}
|
||||
@@ -1,253 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.license.service.impl;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.TypeReference;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.component.RestTool;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.NetUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.km.KmNodeService;
|
||||
import com.xiaojukeji.know.streaming.km.license.service.LicenseService;
|
||||
import com.xiaojukeji.know.streaming.km.license.bean.*;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class LicenseServiceImpl implements LicenseService {
|
||||
private static final ILog LOGGER = LogFactory.getLog(LicenseServiceImpl.class);
|
||||
|
||||
private static final String LICENSE_INFO_URL = "/api/license/info";
|
||||
private static final String LICENSE_USAGE_URL = "/api/license/usage";
|
||||
|
||||
private static final String LICENSE_HEADER_TOKEN = "x-l-token";
|
||||
private static final String LICENSE_HEADER_APP = "x-l-app-name";
|
||||
private static final String LICENSE_HEADER_SIGNATURE = "x-l-signature";
|
||||
|
||||
private static final int FAILED_NO_LICENSE = 1000000000;
|
||||
private static final int FAILED_LICENSE_EXPIRE = 1000000001;
|
||||
private static final int FAILED_LICENSE_CLUSTER_LIMIT = 1000000002;
|
||||
|
||||
private static final int ONE_HOUR = 60 * 60 * 1000;
|
||||
|
||||
@Value("${license.server}")
|
||||
private String licenseSrvUrl;
|
||||
|
||||
@Value("${license.signature}")
|
||||
private String licenseSignature;
|
||||
|
||||
@Value("${license.token}")
|
||||
private String licenseToken;
|
||||
|
||||
@Value("${license.app-name}")
|
||||
private String appName;
|
||||
|
||||
@Autowired
|
||||
private KmNodeService kmNodeService;
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyService clusterPhyService;
|
||||
|
||||
@Autowired
|
||||
private RestTool restTool;
|
||||
|
||||
private LicenseInfo<KmLicense> kmLicense;
|
||||
|
||||
private List<LicenseUsage> licenseUsages = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public Result<Void> addClusterLimit() {
|
||||
//对 LicenseUsage 按照时间挫，从小到大排序，即最新的在最后面
|
||||
licenseUsages.sort((o1, o2) -> o1.getTimeStamp() < o2.getTimeStamp() ? 1 : -1);
|
||||
|
||||
List<KmLicenseUsageDetail> details = licenseUsages.stream()
|
||||
.map(l -> JSON.parseObject(l.getData(), KmLicenseUsageDetail.class))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if(CollectionUtils.isEmpty(details)){return Result.buildSuc();}
|
||||
|
||||
//Tuple.v1 : ks cluster hosts
|
||||
//Tuple.v2 : ks 集群管理的 kafka 集群个数
|
||||
List<Tuple<List<String>, Integer>> ksClusterHostsList = new ArrayList<>();
|
||||
ksClusterHostsList.add(new Tuple<>(details.get(0).getHosts(), details.get(0).getClusters()));
|
||||
|
||||
//根据 hosts 是否有交集，来获取 ks 的集群列表
|
||||
for(KmLicenseUsageDetail detail : details){
|
||||
for(Tuple<List<String>, Integer> tuple : ksClusterHostsList){
|
||||
if(isListIntersection(tuple.getV1(), detail.getHosts())){
|
||||
tuple.setV1(detail.getHosts());
|
||||
tuple.setV2(detail.getClusters());
|
||||
}else {
|
||||
ksClusterHostsList.add(new Tuple<>(detail.getHosts(), detail.getClusters()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LOGGER.debug("method=addClusterLimit||details={}||ksClusterHostsList={}",
|
||||
JSON.toJSONString(details), JSON.toJSONString(ksClusterHostsList));
|
||||
|
||||
//计算索引 ks 集群管理的 kafka 集群总个数
|
||||
final int[] totalKafkaClusterNus = {0};
|
||||
ksClusterHostsList.stream().forEach(l -> totalKafkaClusterNus[0] += l.getV2() );
|
||||
|
||||
if(null == kmLicense) {
|
||||
return Result.buildFailure(FAILED_NO_LICENSE, "无法获取KS的License信息");
|
||||
}
|
||||
|
||||
if(kmLicense.getInfo().getClusters() < totalKafkaClusterNus[0]) {
|
||||
return Result.buildFailure(FAILED_LICENSE_CLUSTER_LIMIT, String.format("KS管理的Kafka集群已达到License限制的%d个集群", kmLicense.getInfo().getClusters()));
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
/**
|
||||
* 当前这个接口只做最小限度的校验，即 km-license 模块和 license 信息存在，
|
||||
* 其他异常情况，如：license-srv 临时挂掉不考虑
|
||||
* check 接口返回的异常 code、msg，就在该模块定义，不要放到 ResultStatus 中
|
||||
*/
|
||||
@Override
|
||||
public Result<Void> check() {
|
||||
if(null == kmLicense){
|
||||
return Result.buildFailure(FAILED_NO_LICENSE, "无法获取KS的license信息");
|
||||
}
|
||||
|
||||
if(System.currentTimeMillis() > kmLicense.getExpiredDate() * 1000){
|
||||
return Result.buildFailure(FAILED_LICENSE_EXPIRE, "当前KS的license已过期");
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void init(){
|
||||
syncLicenseInfo();
|
||||
}
|
||||
|
||||
/**
|
||||
* 每10分钟同步一次
|
||||
*/
|
||||
@Scheduled(cron="0 0/10 * * * ?")
|
||||
public void syncLicenseInfo(){
|
||||
try {
|
||||
saveLicenseUsageInfo();
|
||||
|
||||
List<LicenseUsage> licenseUsages = listLicenseUsageInfo();
|
||||
if(!CollectionUtils.isEmpty(licenseUsages)){
|
||||
this.licenseUsages.clear();
|
||||
this.licenseUsages.addAll(licenseUsages);
|
||||
}
|
||||
|
||||
LicenseInfo<KmLicense> kmLicense = this.getLicenseInfo();
|
||||
if(null != kmLicense){
|
||||
this.kmLicense = kmLicense;
|
||||
}
|
||||
} catch (Exception e){
|
||||
LOGGER.error("method=syncLicenseInfo||msg=exception!", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private LicenseInfo<KmLicense> getLicenseInfo(){
|
||||
String url = licenseSrvUrl + LICENSE_INFO_URL;
|
||||
LicenseResult<String> ret = restTool.getForObject(
|
||||
url, genHeaders(), new TypeReference<LicenseResult<String>>(){});
|
||||
|
||||
LOGGER.debug("method=getLicenseInfo||url={}||ret={}", url, JSON.toJSONString(ret));
|
||||
|
||||
if(!StringUtils.isEmpty(ret.getErr())){
|
||||
return null;
|
||||
}
|
||||
|
||||
byte[] encrypted = Base64.getDecoder().decode(ret.getReply().getBytes(StandardCharsets.UTF_8));
|
||||
LicenseInfo<KmLicense> info = JSON.parseObject(
|
||||
new String(encrypted),
|
||||
new TypeReference<LicenseInfo<KmLicense>>(){}
|
||||
);
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
private List<LicenseUsage> listLicenseUsageInfo(){
|
||||
String url = licenseSrvUrl + LICENSE_USAGE_URL;
|
||||
LicenseResult<List<LicenseUsage>> ret = restTool.getForObject(
|
||||
url, genHeaders(), new TypeReference<LicenseResult<List<LicenseUsage>>>(){});
|
||||
|
||||
LOGGER.debug("method=listLicenseUsageInfo||url={}||ret={}", url, JSON.toJSONString(ret));
|
||||
|
||||
if(!StringUtils.isEmpty(ret.getErr())){
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
List<LicenseUsage> licenseUsages = ret.getReply();
|
||||
if(!CollectionUtils.isEmpty(licenseUsages)){
|
||||
long now = System.currentTimeMillis();
|
||||
|
||||
return licenseUsages.stream()
|
||||
.filter(l -> l.getTimeStamp() + 6 * ONE_HOUR > now)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
private boolean saveLicenseUsageInfo(){
|
||||
String host = NetUtils.localHost();
|
||||
|
||||
KmLicenseUsageDetail detail = new KmLicenseUsageDetail();
|
||||
detail.setHost(host);
|
||||
detail.setHosts(kmNodeService.listKmHosts());
|
||||
detail.setClusters(clusterPhyService.listAllClusters().size());
|
||||
|
||||
LicenseUsage licenseUsage = new LicenseUsage();
|
||||
licenseUsage.setTimeStamp(System.currentTimeMillis());
|
||||
licenseUsage.setUuid(CommonUtils.getMD5(host));
|
||||
licenseUsage.setData(JSON.toJSONString(detail));
|
||||
|
||||
Map<String, String> param = new HashMap<>();
|
||||
param.put("usageSecret", Base64.getEncoder().encodeToString(JSON.toJSONString(licenseUsage).getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
String url = licenseSrvUrl + LICENSE_USAGE_URL;
|
||||
LicenseResult<Void> ret = restTool.putForObject(url, genHeaders(), JSON.toJSONString(param), LicenseResult.class);
|
||||
|
||||
LOGGER.debug("method=saveLicenseUsageInfo||url={}||ret={}", url, JSON.toJSONString(ret));
|
||||
|
||||
if(!StringUtils.isEmpty(ret.getErr())){
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private HttpHeaders genHeaders(){
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add(LICENSE_HEADER_TOKEN, licenseToken);
|
||||
headers.add(LICENSE_HEADER_APP, appName);
|
||||
headers.add(LICENSE_HEADER_SIGNATURE, licenseSignature);
|
||||
headers.add("content-type", "application/json");
|
||||
return headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* 两个 list 是否相交，是否有相同的内容
|
||||
* @return
|
||||
*/
|
||||
private boolean isListIntersection(List<String> l, List<String> r){
|
||||
l.retainAll(r);
|
||||
return !CollectionUtils.isEmpty(l);
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>km</artifactId>
|
||||
<groupId>com.xiaojukeji.kafka</groupId>
|
||||
<version>${km.revision}</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>km-rebalance</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.sf.jopt-simple</groupId>
|
||||
<artifactId>jopt-simple</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<version>1.2</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>-->
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -1,139 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance;
|
||||
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.HostEnv;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.utils.CommandLineUtils;
|
||||
import joptsimple.OptionParser;
|
||||
import joptsimple.OptionSet;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.kafka.clients.CommonClientConfigs;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
public class KafkaRebalanceMain {
|
||||
|
||||
public void run(OptionSet options) {
|
||||
try {
|
||||
BalanceParameter balanceParameter = new BalanceParameter();
|
||||
if (options.has("excluded-topics")) {
|
||||
balanceParameter.setExcludedTopics(options.valueOf("excluded-topics").toString());
|
||||
}
|
||||
if (options.has("offline-brokers")) {
|
||||
balanceParameter.setOfflineBrokers(options.valueOf("offline-brokers").toString());
|
||||
}
|
||||
if (options.has("disk-threshold")) {
|
||||
Double diskThreshold = (Double) options.valueOf("disk-threshold");
|
||||
balanceParameter.setDiskThreshold(diskThreshold);
|
||||
}
|
||||
if (options.has("cpu-threshold")) {
|
||||
Double cpuThreshold = (Double) options.valueOf("cpu-threshold");
|
||||
balanceParameter.setCpuThreshold(cpuThreshold);
|
||||
}
|
||||
if (options.has("network-in-threshold")) {
|
||||
Double networkInThreshold = (Double) options.valueOf("network-in-threshold");
|
||||
balanceParameter.setNetworkInThreshold(networkInThreshold);
|
||||
}
|
||||
if (options.has("network-out-threshold")) {
|
||||
Double networkOutThreshold = (Double) options.valueOf("network-out-threshold");
|
||||
balanceParameter.setNetworkOutThreshold(networkOutThreshold);
|
||||
}
|
||||
if (options.has("balance-brokers")) {
|
||||
balanceParameter.setBalanceBrokers(options.valueOf("balance-brokers").toString());
|
||||
}
|
||||
if (options.has("topic-leader-threshold")) {
|
||||
Double topicLeaderThreshold = (Double) options.valueOf("topic-leader-threshold");
|
||||
balanceParameter.setTopicLeaderThreshold(topicLeaderThreshold);
|
||||
}
|
||||
if (options.has("topic-replica-threshold")) {
|
||||
Double topicReplicaThreshold = (Double) options.valueOf("topic-replica-threshold");
|
||||
balanceParameter.setTopicReplicaThreshold(topicReplicaThreshold);
|
||||
}
|
||||
if (options.has("ignored-topics")) {
|
||||
balanceParameter.setIgnoredTopics(options.valueOf("ignored-topics").toString());
|
||||
}
|
||||
String path = options.valueOf("output-path").toString();
|
||||
String goals = options.valueOf("goals").toString();
|
||||
balanceParameter.setGoals(Arrays.asList(goals.split(",")));
|
||||
balanceParameter.setCluster(options.valueOf("cluster").toString());
|
||||
Properties kafkaConfig = new Properties();
|
||||
kafkaConfig.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, options.valueOf("bootstrap-servers").toString());
|
||||
balanceParameter.setKafkaConfig(kafkaConfig);
|
||||
balanceParameter.setEsRestURL(options.valueOf("es-rest-url").toString());
|
||||
balanceParameter.setEsIndexPrefix(options.valueOf("es-index-prefix").toString());
|
||||
balanceParameter.setBeforeSeconds((Integer) options.valueOf("before-seconds"));
|
||||
String envFile = options.valueOf("hardware-env-file").toString();
|
||||
String envJson = FileUtils.readFileToString(new File(envFile), "UTF-8");
|
||||
List<HostEnv> env = new ObjectMapper().readValue(envJson, new TypeReference<List<HostEnv>>() {
|
||||
});
|
||||
balanceParameter.setHardwareEnv(env);
|
||||
ExecutionRebalance exec = new ExecutionRebalance();
|
||||
OptimizerResult optimizerResult = exec.optimizations(balanceParameter);
|
||||
FileUtils.write(new File(path.concat("/overview.json")), optimizerResult.resultJsonOverview(), "UTF-8");
|
||||
FileUtils.write(new File(path.concat("/detailed.json")), optimizerResult.resultJsonDetailed(), "UTF-8");
|
||||
FileUtils.write(new File(path.concat("/task.json")), optimizerResult.resultJsonTask(), "UTF-8");
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
OptionParser parser = new OptionParser();
|
||||
parser.accepts("bootstrap-servers", "Kafka cluster boot server").withRequiredArg().ofType(String.class);
|
||||
parser.accepts("es-rest-url", "The url of elasticsearch").withRequiredArg().ofType(String.class);
|
||||
parser.accepts("es-index-prefix", "The Index Prefix of elasticsearch").withRequiredArg().ofType(String.class);
|
||||
parser.accepts("goals", "Balanced goals include TopicLeadersDistributionGoal,TopicReplicaDistributionGoal,DiskDistributionGoal,NetworkInboundDistributionGoal,NetworkOutboundDistributionGoal").withRequiredArg().ofType(String.class);
|
||||
parser.accepts("cluster", "Balanced cluster name").withRequiredArg().ofType(String.class);
|
||||
parser.accepts("excluded-topics", "Topic does not perform data balancing").withOptionalArg().ofType(String.class);
|
||||
parser.accepts("ignored-topics","Topics that do not contain model calculations").withOptionalArg().ofType(String.class);
|
||||
parser.accepts("offline-brokers", "Broker does not perform data balancing").withOptionalArg().ofType(String.class);
|
||||
parser.accepts("balance-brokers", "Balanced brokers list").withOptionalArg().ofType(String.class);
|
||||
parser.accepts("disk-threshold", "Disk data balance threshold").withOptionalArg().ofType(Double.class);
|
||||
parser.accepts("topic-leader-threshold","topic leader threshold").withOptionalArg().ofType(Double.class);
|
||||
parser.accepts("topic-replica-threshold","topic replica threshold").withOptionalArg().ofType(Double.class);
|
||||
parser.accepts("cpu-threshold", "Cpu utilization balance threshold").withOptionalArg().ofType(Double.class);
|
||||
parser.accepts("network-in-threshold", "Network inflow threshold").withOptionalArg().ofType(Double.class);
|
||||
parser.accepts("network-out-threshold", "Network outflow threshold").withOptionalArg().ofType(Double.class);
|
||||
parser.accepts("before-seconds", "Query es data time").withRequiredArg().ofType(Integer.class);
|
||||
parser.accepts("hardware-env-file", "Machine environment information includes cpu, disk and network").withRequiredArg().ofType(String.class);
|
||||
parser.accepts("output-path", "Cluster balancing result file directory").withRequiredArg().ofType(String.class);
|
||||
OptionSet options = parser.parse(args);
|
||||
if (args.length == 0) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "Running parameters need to be configured to perform cluster balancing");
|
||||
}
|
||||
if (!options.has("bootstrap-servers")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "bootstrap-servers cannot be empty");
|
||||
}
|
||||
if (!options.has("es-rest-url")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "es-rest-url cannot be empty");
|
||||
}
|
||||
if (!options.has("es-index-prefix")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "es-index-prefix cannot be empty");
|
||||
}
|
||||
if (!options.has("goals")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "goals cannot be empty");
|
||||
}
|
||||
if (!options.has("cluster")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "cluster name cannot be empty");
|
||||
}
|
||||
if (!options.has("before-seconds")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "before-seconds cannot be empty");
|
||||
}
|
||||
if (!options.has("hardware-env-file")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "hardware-env-file cannot be empty");
|
||||
}
|
||||
if (!options.has("output-path")) {
|
||||
CommandLineUtils.printUsageAndDie(parser, "output-path cannot be empty");
|
||||
}
|
||||
KafkaRebalanceMain rebalanceMain = new KafkaRebalanceMain();
|
||||
rebalanceMain.run(options);
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.exception;
|
||||
|
||||
public class OptimizationFailureException extends Exception {
|
||||
public OptimizationFailureException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public OptimizationFailureException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public OptimizationFailureException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceGoal;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceThreshold;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BrokerBalanceState;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Load;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.GoalOptimizer;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class ExecutionRebalance {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ExecutionRebalance.class);
|
||||
|
||||
public OptimizerResult optimizations(BalanceParameter balanceParameter) {
|
||||
Validate.isTrue(StringUtils.isNotBlank(balanceParameter.getCluster()), "cluster is empty");
|
||||
Validate.isTrue(balanceParameter.getKafkaConfig() != null, "Kafka config properties is empty");
|
||||
Validate.isTrue(balanceParameter.getGoals() != null, "Balance goals is empty");
|
||||
Validate.isTrue(StringUtils.isNotBlank(balanceParameter.getEsIndexPrefix()), "EsIndexPrefix is empty");
|
||||
Validate.isTrue(StringUtils.isNotBlank(balanceParameter.getEsRestURL()), "EsRestURL is empty");
|
||||
Validate.isTrue(balanceParameter.getHardwareEnv() != null, "HardwareEnv is empty");
|
||||
logger.info("Cluster balancing start");
|
||||
ClusterModel clusterModel = GoalUtils.getInitClusterModel(balanceParameter);
|
||||
GoalOptimizer optimizer = new GoalOptimizer();
|
||||
OptimizerResult optimizerResult = optimizer.optimizations(clusterModel, new OptimizationOptions(balanceParameter));
|
||||
logger.info("Cluster balancing completed");
|
||||
return optimizerResult;
|
||||
}
|
||||
|
||||
public static Map<Resource, Double> getClusterAvgResourcesState(BalanceParameter balanceParameter) {
|
||||
ClusterModel clusterModel = GoalUtils.getInitClusterModel(balanceParameter);
|
||||
Load load = clusterModel.load();
|
||||
Map<Resource, Double> avgResource = new HashMap<>();
|
||||
avgResource.put(Resource.DISK, load.loadFor(Resource.DISK) / clusterModel.brokers().size());
|
||||
avgResource.put(Resource.CPU, load.loadFor(Resource.CPU) / clusterModel.brokers().size());
|
||||
avgResource.put(Resource.NW_OUT, load.loadFor(Resource.NW_OUT) / clusterModel.brokers().size());
|
||||
avgResource.put(Resource.NW_IN, load.loadFor(Resource.NW_IN) / clusterModel.brokers().size());
|
||||
return avgResource;
|
||||
}
|
||||
|
||||
public static Map<Integer, BrokerBalanceState> getBrokerResourcesBalanceState(BalanceParameter balanceParameter) {
|
||||
Map<Integer, BrokerBalanceState> balanceState = new HashMap<>();
|
||||
ClusterModel clusterModel = GoalUtils.getInitClusterModel(balanceParameter);
|
||||
double[] clusterAvgResource = clusterModel.avgOfUtilization();
|
||||
Map<String, BalanceThreshold> balanceThreshold = GoalUtils.getBalanceThreshold(balanceParameter, clusterAvgResource);
|
||||
clusterModel.brokers().forEach(i -> {
|
||||
BrokerBalanceState state = new BrokerBalanceState();
|
||||
if (balanceParameter.getGoals().contains(BalanceGoal.DISK.goal())) {
|
||||
state.setDiskAvgResource(i.load().loadFor(Resource.DISK));
|
||||
state.setDiskUtilization(i.utilizationFor(Resource.DISK));
|
||||
state.setDiskBalanceState(balanceThreshold.get(BalanceGoal.DISK.goal()).state(i.utilizationFor(Resource.DISK)));
|
||||
}
|
||||
if (balanceParameter.getGoals().contains(BalanceGoal.NW_IN.goal())) {
|
||||
state.setBytesInAvgResource(i.load().loadFor(Resource.NW_IN));
|
||||
state.setBytesInUtilization(i.utilizationFor(Resource.NW_IN));
|
||||
state.setBytesInBalanceState(balanceThreshold.get(BalanceGoal.NW_IN.goal()).state(i.utilizationFor(Resource.NW_IN)));
|
||||
}
|
||||
if (balanceParameter.getGoals().contains(BalanceGoal.NW_OUT.goal())) {
|
||||
state.setBytesOutAvgResource(i.load().loadFor(Resource.NW_OUT));
|
||||
state.setBytesOutUtilization(i.utilizationFor(Resource.NW_OUT));
|
||||
state.setBytesOutBalanceState(balanceThreshold.get(BalanceGoal.NW_OUT.goal()).state(i.utilizationFor(Resource.NW_OUT)));
|
||||
}
|
||||
balanceState.put(i.id(), state);
|
||||
});
|
||||
return balanceState;
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
public class BalanceActionHistory {
|
||||
//均衡目标
|
||||
private String goal;
|
||||
//均衡动作
|
||||
private String actionType;
|
||||
//均衡Topic
|
||||
private String topic;
|
||||
//均衡分区
|
||||
private int partition;
|
||||
//源Broker
|
||||
private int sourceBrokerId;
|
||||
//目标Broker
|
||||
private int destinationBrokerId;
|
||||
|
||||
public String getGoal() {
|
||||
return goal;
|
||||
}
|
||||
|
||||
public void setGoal(String goal) {
|
||||
this.goal = goal;
|
||||
}
|
||||
|
||||
public String getActionType() {
|
||||
return actionType;
|
||||
}
|
||||
|
||||
public void setActionType(String actionType) {
|
||||
this.actionType = actionType;
|
||||
}
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public int getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(int partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
public int getSourceBrokerId() {
|
||||
return sourceBrokerId;
|
||||
}
|
||||
|
||||
public void setSourceBrokerId(int sourceBrokerId) {
|
||||
this.sourceBrokerId = sourceBrokerId;
|
||||
}
|
||||
|
||||
public int getDestinationBrokerId() {
|
||||
return destinationBrokerId;
|
||||
}
|
||||
|
||||
public void setDestinationBrokerId(int destinationBrokerId) {
|
||||
this.destinationBrokerId = destinationBrokerId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BalanceActionHistory{" +
|
||||
"goal='" + goal + '\'' +
|
||||
", actionType='" + actionType + '\'' +
|
||||
", topic='" + topic + '\'' +
|
||||
", partition='" + partition + '\'' +
|
||||
", sourceBrokerId='" + sourceBrokerId + '\'' +
|
||||
", destinationBrokerId='" + destinationBrokerId + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
public class BalanceDetailed {
|
||||
private int brokerId;
|
||||
private String host;
|
||||
//当前CPU使用率
|
||||
private double currentCPUUtilization;
|
||||
//最新CPU使用率
|
||||
private double lastCPUUtilization;
|
||||
//当前磁盘使用率
|
||||
private double currentDiskUtilization;
|
||||
//最新磁盘使用量
|
||||
private double lastDiskUtilization;
|
||||
//当前网卡入流量
|
||||
private double currentNetworkInUtilization;
|
||||
//最新网卡入流量
|
||||
private double lastNetworkInUtilization;
|
||||
//当前网卡出流量
|
||||
private double currentNetworkOutUtilization;
|
||||
//最新网卡出流量
|
||||
private double lastNetworkOutUtilization;
|
||||
//均衡状态
|
||||
private int balanceState = 0;
|
||||
//迁入磁盘容量
|
||||
private double moveInDiskSize;
|
||||
//迁出磁盘容量
|
||||
private double moveOutDiskSize;
|
||||
//迁入副本数
|
||||
private double moveInReplicas;
|
||||
//迁出副本数
|
||||
private double moveOutReplicas;
|
||||
|
||||
public int getBrokerId() {
|
||||
return brokerId;
|
||||
}
|
||||
|
||||
public void setBrokerId(int brokerId) {
|
||||
this.brokerId = brokerId;
|
||||
}
|
||||
|
||||
public double getCurrentCPUUtilization() {
|
||||
return currentCPUUtilization;
|
||||
}
|
||||
|
||||
public void setCurrentCPUUtilization(double currentCPUUtilization) {
|
||||
this.currentCPUUtilization = currentCPUUtilization;
|
||||
}
|
||||
|
||||
public double getLastCPUUtilization() {
|
||||
return lastCPUUtilization;
|
||||
}
|
||||
|
||||
public void setLastCPUUtilization(double lastCPUUtilization) {
|
||||
this.lastCPUUtilization = lastCPUUtilization;
|
||||
}
|
||||
|
||||
public double getCurrentDiskUtilization() {
|
||||
return currentDiskUtilization;
|
||||
}
|
||||
|
||||
public void setCurrentDiskUtilization(double currentDiskUtilization) {
|
||||
this.currentDiskUtilization = currentDiskUtilization;
|
||||
}
|
||||
|
||||
public double getLastDiskUtilization() {
|
||||
return lastDiskUtilization;
|
||||
}
|
||||
|
||||
public void setLastDiskUtilization(double lastDiskUtilization) {
|
||||
this.lastDiskUtilization = lastDiskUtilization;
|
||||
}
|
||||
|
||||
public double getCurrentNetworkInUtilization() {
|
||||
return currentNetworkInUtilization;
|
||||
}
|
||||
|
||||
public void setCurrentNetworkInUtilization(double currentNetworkInUtilization) {
|
||||
this.currentNetworkInUtilization = currentNetworkInUtilization;
|
||||
}
|
||||
|
||||
public double getLastNetworkInUtilization() {
|
||||
return lastNetworkInUtilization;
|
||||
}
|
||||
|
||||
public void setLastNetworkInUtilization(double lastNetworkInUtilization) {
|
||||
this.lastNetworkInUtilization = lastNetworkInUtilization;
|
||||
}
|
||||
|
||||
public double getCurrentNetworkOutUtilization() {
|
||||
return currentNetworkOutUtilization;
|
||||
}
|
||||
|
||||
public void setCurrentNetworkOutUtilization(double currentNetworkOutUtilization) {
|
||||
this.currentNetworkOutUtilization = currentNetworkOutUtilization;
|
||||
}
|
||||
|
||||
public double getLastNetworkOutUtilization() {
|
||||
return lastNetworkOutUtilization;
|
||||
}
|
||||
|
||||
public void setLastNetworkOutUtilization(double lastNetworkOutUtilization) {
|
||||
this.lastNetworkOutUtilization = lastNetworkOutUtilization;
|
||||
}
|
||||
|
||||
public int getBalanceState() {
|
||||
return balanceState;
|
||||
}
|
||||
|
||||
public void setBalanceState(int balanceState) {
|
||||
this.balanceState = balanceState;
|
||||
}
|
||||
|
||||
public double getMoveInDiskSize() {
|
||||
return moveInDiskSize;
|
||||
}
|
||||
|
||||
public void setMoveInDiskSize(double moveInDiskSize) {
|
||||
this.moveInDiskSize = moveInDiskSize;
|
||||
}
|
||||
|
||||
public double getMoveOutDiskSize() {
|
||||
return moveOutDiskSize;
|
||||
}
|
||||
|
||||
public void setMoveOutDiskSize(double moveOutDiskSize) {
|
||||
this.moveOutDiskSize = moveOutDiskSize;
|
||||
}
|
||||
|
||||
public double getMoveInReplicas() {
|
||||
return moveInReplicas;
|
||||
}
|
||||
|
||||
public void setMoveInReplicas(double moveInReplicas) {
|
||||
this.moveInReplicas = moveInReplicas;
|
||||
}
|
||||
|
||||
public double getMoveOutReplicas() {
|
||||
return moveOutReplicas;
|
||||
}
|
||||
|
||||
public void setMoveOutReplicas(double moveOutReplicas) {
|
||||
this.moveOutReplicas = moveOutReplicas;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BalanceDetailed{" +
|
||||
"brokerId=" + brokerId +
|
||||
", host='" + host + '\'' +
|
||||
", currentCPUUtilization=" + currentCPUUtilization +
|
||||
", lastCPUUtilization=" + lastCPUUtilization +
|
||||
", currentDiskUtilization=" + currentDiskUtilization +
|
||||
", lastDiskUtilization=" + lastDiskUtilization +
|
||||
", currentNetworkInUtilization=" + currentNetworkInUtilization +
|
||||
", lastNetworkInUtilization=" + lastNetworkInUtilization +
|
||||
", currentNetworkOutUtilization=" + currentNetworkOutUtilization +
|
||||
", lastNetworkOutUtilization=" + lastNetworkOutUtilization +
|
||||
", balanceState=" + balanceState +
|
||||
", moveInDiskSize=" + moveInDiskSize +
|
||||
", moveOutDiskSize=" + moveOutDiskSize +
|
||||
", moveInReplicas=" + moveInReplicas +
|
||||
", moveOutReplicas=" + moveOutReplicas +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
public enum BalanceGoal {
|
||||
// KM传参时使用
|
||||
TOPIC_LEADERS("TopicLeadersDistributionGoal"),
|
||||
TOPIC_REPLICA("TopicReplicaDistributionGoal"),
|
||||
DISK("DiskDistributionGoal"),
|
||||
NW_IN("NetworkInboundDistributionGoal"),
|
||||
NW_OUT("NetworkOutboundDistributionGoal");
|
||||
|
||||
private final String goal;
|
||||
|
||||
BalanceGoal(String goal) {
|
||||
this.goal = goal;
|
||||
}
|
||||
|
||||
public String goal() {
|
||||
return goal;
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class BalanceOverview {
|
||||
//任务类型
|
||||
private String taskType;
|
||||
//节点范围
|
||||
private String nodeRange;
|
||||
//总的迁移大小
|
||||
private double totalMoveSize;
|
||||
//topic黑名单
|
||||
private String topicBlacklist;
|
||||
//迁移副本数
|
||||
private int moveReplicas;
|
||||
//迁移Topic
|
||||
private String moveTopics;
|
||||
//均衡阈值
|
||||
private Map<Resource, Double> balanceThreshold;
|
||||
//移除节点
|
||||
private String removeNode;
|
||||
|
||||
public String getTaskType() {
|
||||
return taskType;
|
||||
}
|
||||
|
||||
public void setTaskType(String taskType) {
|
||||
this.taskType = taskType;
|
||||
}
|
||||
|
||||
public String getNodeRange() {
|
||||
return nodeRange;
|
||||
}
|
||||
|
||||
public void setNodeRange(String nodeRange) {
|
||||
this.nodeRange = nodeRange;
|
||||
}
|
||||
|
||||
public double getTotalMoveSize() {
|
||||
return totalMoveSize;
|
||||
}
|
||||
|
||||
public void setTotalMoveSize(double totalMoveSize) {
|
||||
this.totalMoveSize = totalMoveSize;
|
||||
}
|
||||
|
||||
public String getTopicBlacklist() {
|
||||
return topicBlacklist;
|
||||
}
|
||||
|
||||
public void setTopicBlacklist(String topicBlacklist) {
|
||||
this.topicBlacklist = topicBlacklist;
|
||||
}
|
||||
|
||||
public int getMoveReplicas() {
|
||||
return moveReplicas;
|
||||
}
|
||||
|
||||
public void setMoveReplicas(int moveReplicas) {
|
||||
this.moveReplicas = moveReplicas;
|
||||
}
|
||||
|
||||
public String getMoveTopics() {
|
||||
return moveTopics;
|
||||
}
|
||||
|
||||
public void setMoveTopics(String moveTopics) {
|
||||
this.moveTopics = moveTopics;
|
||||
}
|
||||
|
||||
public Map<Resource, Double> getBalanceThreshold() {
|
||||
return balanceThreshold;
|
||||
}
|
||||
|
||||
public void setBalanceThreshold(Map<Resource, Double> balanceThreshold) {
|
||||
this.balanceThreshold = balanceThreshold;
|
||||
}
|
||||
|
||||
public String getRemoveNode() {
|
||||
return removeNode;
|
||||
}
|
||||
|
||||
public void setRemoveNode(String removeNode) {
|
||||
this.removeNode = removeNode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BalanceOverview{" +
|
||||
"taskType='" + taskType + '\'' +
|
||||
", nodeRange='" + nodeRange + '\'' +
|
||||
", totalMoveSize=" + totalMoveSize +
|
||||
", topicBlacklist='" + topicBlacklist + '\'' +
|
||||
", moveReplicas=" + moveReplicas +
|
||||
", moveTopics='" + moveTopics + '\'' +
|
||||
", balanceThreshold=" + balanceThreshold +
|
||||
", removeNode='" + removeNode + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
public class BalanceParameter {
|
||||
//集群名称
|
||||
private String cluster;
|
||||
//集群访问配置
|
||||
private Properties kafkaConfig;
|
||||
//ES访问地址
|
||||
private String esRestURL;
|
||||
//ES存储索引前缀
|
||||
private String esIndexPrefix;
|
||||
//均衡目标
|
||||
private List<String> goals;
|
||||
//Topic黑名单，（参与模型计算）
|
||||
private String excludedTopics = "";
|
||||
//忽略的Topic列表，（不参与模型计算）
|
||||
private String ignoredTopics = "";
|
||||
//下线的Broker
|
||||
private String offlineBrokers = "";
|
||||
//需要均衡的Broker
|
||||
private String balanceBrokers = "";
|
||||
//默认Topic副本分布阈值
|
||||
private double topicReplicaThreshold = 0.1;
|
||||
//磁盘浮动阈值
|
||||
private double diskThreshold = 0.1;
|
||||
//CPU浮动阈值
|
||||
private double cpuThreshold = 0.1;
|
||||
//流入浮动阈值
|
||||
private double networkInThreshold = 0.1;
|
||||
//流出浮动阈值
|
||||
private double networkOutThreshold = 0.1;
|
||||
//均衡时间范围
|
||||
private int beforeSeconds = 300;
|
||||
//集群中所有Broker的硬件环境:cpu、disk、bytesIn、bytesOut
|
||||
private List<HostEnv> hardwareEnv;
|
||||
//最小Leader浮动阈值,不追求绝对平均，避免集群流量抖动
|
||||
private double topicLeaderThreshold = 0.1;
|
||||
|
||||
public String getCluster() {
|
||||
return cluster;
|
||||
}
|
||||
|
||||
public void setCluster(String cluster) {
|
||||
this.cluster = cluster;
|
||||
}
|
||||
|
||||
public String getEsRestURL() {
|
||||
return esRestURL;
|
||||
}
|
||||
|
||||
public void setEsRestURL(String esRestURL) {
|
||||
this.esRestURL = esRestURL;
|
||||
}
|
||||
|
||||
public List<String> getGoals() {
|
||||
return goals;
|
||||
}
|
||||
|
||||
public void setGoals(List<String> goals) {
|
||||
this.goals = goals;
|
||||
}
|
||||
|
||||
public String getExcludedTopics() {
|
||||
return excludedTopics;
|
||||
}
|
||||
|
||||
public void setExcludedTopics(String excludedTopics) {
|
||||
this.excludedTopics = excludedTopics;
|
||||
}
|
||||
|
||||
public String getIgnoredTopics() {
|
||||
return ignoredTopics;
|
||||
}
|
||||
|
||||
public void setIgnoredTopics(String ignoredTopics) {
|
||||
this.ignoredTopics = ignoredTopics;
|
||||
}
|
||||
|
||||
public double getTopicReplicaThreshold() {
|
||||
return topicReplicaThreshold;
|
||||
}
|
||||
|
||||
public void setTopicReplicaThreshold(double topicReplicaThreshold) {
|
||||
this.topicReplicaThreshold = topicReplicaThreshold;
|
||||
}
|
||||
|
||||
public double getDiskThreshold() {
|
||||
return diskThreshold;
|
||||
}
|
||||
|
||||
public void setDiskThreshold(double diskThreshold) {
|
||||
this.diskThreshold = diskThreshold;
|
||||
}
|
||||
|
||||
public double getCpuThreshold() {
|
||||
return cpuThreshold;
|
||||
}
|
||||
|
||||
public void setCpuThreshold(double cpuThreshold) {
|
||||
this.cpuThreshold = cpuThreshold;
|
||||
}
|
||||
|
||||
public double getNetworkInThreshold() {
|
||||
return networkInThreshold;
|
||||
}
|
||||
|
||||
public void setNetworkInThreshold(double networkInThreshold) {
|
||||
this.networkInThreshold = networkInThreshold;
|
||||
}
|
||||
|
||||
public double getNetworkOutThreshold() {
|
||||
return networkOutThreshold;
|
||||
}
|
||||
|
||||
public void setNetworkOutThreshold(double networkOutThreshold) {
|
||||
this.networkOutThreshold = networkOutThreshold;
|
||||
}
|
||||
|
||||
public List<HostEnv> getHardwareEnv() {
|
||||
return hardwareEnv;
|
||||
}
|
||||
|
||||
public void setHardwareEnv(List<HostEnv> hardwareEnv) {
|
||||
this.hardwareEnv = hardwareEnv;
|
||||
}
|
||||
|
||||
public String getBalanceBrokers() {
|
||||
return balanceBrokers;
|
||||
}
|
||||
|
||||
public void setBalanceBrokers(String balanceBrokers) {
|
||||
this.balanceBrokers = balanceBrokers;
|
||||
}
|
||||
|
||||
public Properties getKafkaConfig() {
|
||||
return kafkaConfig;
|
||||
}
|
||||
|
||||
public void setKafkaConfig(Properties kafkaConfig) {
|
||||
this.kafkaConfig = kafkaConfig;
|
||||
}
|
||||
|
||||
public String getEsIndexPrefix() {
|
||||
return esIndexPrefix;
|
||||
}
|
||||
|
||||
public void setEsIndexPrefix(String esIndexPrefix) {
|
||||
this.esIndexPrefix = esIndexPrefix;
|
||||
}
|
||||
|
||||
public String getOfflineBrokers() {
|
||||
return offlineBrokers;
|
||||
}
|
||||
|
||||
public void setOfflineBrokers(String offlineBrokers) {
|
||||
this.offlineBrokers = offlineBrokers;
|
||||
}
|
||||
|
||||
public int getBeforeSeconds() {
|
||||
return beforeSeconds;
|
||||
}
|
||||
|
||||
public void setBeforeSeconds(int beforeSeconds) {
|
||||
this.beforeSeconds = beforeSeconds;
|
||||
}
|
||||
|
||||
public double getTopicLeaderThreshold() {
|
||||
return topicLeaderThreshold;
|
||||
}
|
||||
|
||||
public void setTopicLeaderThreshold(double topicLeaderThreshold) {
|
||||
this.topicLeaderThreshold = topicLeaderThreshold;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BalanceParameter{" +
|
||||
"cluster='" + cluster + '\'' +
|
||||
", kafkaConfig=" + kafkaConfig +
|
||||
", esRestURL='" + esRestURL + '\'' +
|
||||
", esIndexPrefix='" + esIndexPrefix + '\'' +
|
||||
", goals=" + goals +
|
||||
", excludedTopics='" + excludedTopics + '\'' +
|
||||
", offlineBrokers='" + offlineBrokers + '\'' +
|
||||
", balanceBrokers='" + balanceBrokers + '\'' +
|
||||
", topicReplicaThreshold=" + topicReplicaThreshold +
|
||||
", diskThreshold=" + diskThreshold +
|
||||
", cpuThreshold=" + cpuThreshold +
|
||||
", networkInThreshold=" + networkInThreshold +
|
||||
", networkOutThreshold=" + networkOutThreshold +
|
||||
", beforeSeconds=" + beforeSeconds +
|
||||
", hardwareEnv=" + hardwareEnv +
|
||||
", topicLeaderThreshold=" + topicLeaderThreshold +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class BalanceTask {
|
||||
private String topic;
|
||||
private int partition;
|
||||
//副本分配列表
|
||||
private List<Integer> replicas;
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public int getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(int partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
public List<Integer> getReplicas() {
|
||||
return replicas;
|
||||
}
|
||||
|
||||
public void setReplicas(List<Integer> replicas) {
|
||||
this.replicas = replicas;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BalanceTask{" +
|
||||
"topic='" + topic + '\'' +
|
||||
", partition=" + partition +
|
||||
", replicas=" + replicas +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
|
||||
public class BalanceThreshold {
|
||||
private final Resource _resource;
|
||||
private final double _upper;
|
||||
private final double _lower;
|
||||
|
||||
public BalanceThreshold(Resource resource, double threshold, double avgResource) {
|
||||
_resource = resource;
|
||||
_upper = avgResource * (1 + threshold);
|
||||
_lower = avgResource * (1 - threshold);
|
||||
}
|
||||
|
||||
public Resource resource() {
|
||||
return _resource;
|
||||
}
|
||||
|
||||
public boolean isInRange(double utilization) {
|
||||
return utilization > _lower && utilization < _upper;
|
||||
}
|
||||
|
||||
public int state(double utilization) {
|
||||
if (utilization <= _lower) {
|
||||
return -1;
|
||||
} else if (utilization >= _upper) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BalanceThreshold{" +
|
||||
"_resource=" + _resource +
|
||||
", _upper=" + _upper +
|
||||
", _lower=" + _lower +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
public class BrokerBalanceState {
|
||||
//CPU平均资源
|
||||
private Double cpuAvgResource;
|
||||
//CPU资源使用率
|
||||
private Double cpuUtilization;
|
||||
// -1,低于均衡范围
|
||||
// 0,均衡范围内
|
||||
// 1,高于均衡范围
|
||||
private Integer cpuBalanceState;
|
||||
//磁盘平均资源
|
||||
private Double diskAvgResource;
|
||||
//磁盘资源使用率
|
||||
private Double diskUtilization;
|
||||
//磁盘均衡状态
|
||||
private Integer diskBalanceState;
|
||||
//流入平均资源
|
||||
private Double bytesInAvgResource;
|
||||
//流入资源使用率
|
||||
private Double bytesInUtilization;
|
||||
//流入均衡状态
|
||||
private Integer bytesInBalanceState;
|
||||
//流出平均资源
|
||||
private Double bytesOutAvgResource;
|
||||
//流出资源使用率
|
||||
private Double bytesOutUtilization;
|
||||
//流出均衡状态
|
||||
private Integer bytesOutBalanceState;
|
||||
|
||||
public Double getCpuAvgResource() {
|
||||
return cpuAvgResource;
|
||||
}
|
||||
|
||||
public void setCpuAvgResource(Double cpuAvgResource) {
|
||||
this.cpuAvgResource = cpuAvgResource;
|
||||
}
|
||||
|
||||
public Double getCpuUtilization() {
|
||||
return cpuUtilization;
|
||||
}
|
||||
|
||||
public void setCpuUtilization(Double cpuUtilization) {
|
||||
this.cpuUtilization = cpuUtilization;
|
||||
}
|
||||
|
||||
public Integer getCpuBalanceState() {
|
||||
return cpuBalanceState;
|
||||
}
|
||||
|
||||
public void setCpuBalanceState(Integer cpuBalanceState) {
|
||||
this.cpuBalanceState = cpuBalanceState;
|
||||
}
|
||||
|
||||
public Double getDiskAvgResource() {
|
||||
return diskAvgResource;
|
||||
}
|
||||
|
||||
public void setDiskAvgResource(Double diskAvgResource) {
|
||||
this.diskAvgResource = diskAvgResource;
|
||||
}
|
||||
|
||||
public Double getDiskUtilization() {
|
||||
return diskUtilization;
|
||||
}
|
||||
|
||||
public void setDiskUtilization(Double diskUtilization) {
|
||||
this.diskUtilization = diskUtilization;
|
||||
}
|
||||
|
||||
public Integer getDiskBalanceState() {
|
||||
return diskBalanceState;
|
||||
}
|
||||
|
||||
public void setDiskBalanceState(Integer diskBalanceState) {
|
||||
this.diskBalanceState = diskBalanceState;
|
||||
}
|
||||
|
||||
public Double getBytesInAvgResource() {
|
||||
return bytesInAvgResource;
|
||||
}
|
||||
|
||||
public void setBytesInAvgResource(Double bytesInAvgResource) {
|
||||
this.bytesInAvgResource = bytesInAvgResource;
|
||||
}
|
||||
|
||||
public Double getBytesInUtilization() {
|
||||
return bytesInUtilization;
|
||||
}
|
||||
|
||||
public void setBytesInUtilization(Double bytesInUtilization) {
|
||||
this.bytesInUtilization = bytesInUtilization;
|
||||
}
|
||||
|
||||
public Integer getBytesInBalanceState() {
|
||||
return bytesInBalanceState;
|
||||
}
|
||||
|
||||
public void setBytesInBalanceState(Integer bytesInBalanceState) {
|
||||
this.bytesInBalanceState = bytesInBalanceState;
|
||||
}
|
||||
|
||||
public Double getBytesOutAvgResource() {
|
||||
return bytesOutAvgResource;
|
||||
}
|
||||
|
||||
public void setBytesOutAvgResource(Double bytesOutAvgResource) {
|
||||
this.bytesOutAvgResource = bytesOutAvgResource;
|
||||
}
|
||||
|
||||
public Double getBytesOutUtilization() {
|
||||
return bytesOutUtilization;
|
||||
}
|
||||
|
||||
public void setBytesOutUtilization(Double bytesOutUtilization) {
|
||||
this.bytesOutUtilization = bytesOutUtilization;
|
||||
}
|
||||
|
||||
public Integer getBytesOutBalanceState() {
|
||||
return bytesOutBalanceState;
|
||||
}
|
||||
|
||||
public void setBytesOutBalanceState(Integer bytesOutBalanceState) {
|
||||
this.bytesOutBalanceState = bytesOutBalanceState;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BrokerBalanceState{" +
|
||||
"cpuAvgResource=" + cpuAvgResource +
|
||||
", cpuUtilization=" + cpuUtilization +
|
||||
", cpuBalanceState=" + cpuBalanceState +
|
||||
", diskAvgResource=" + diskAvgResource +
|
||||
", diskUtilization=" + diskUtilization +
|
||||
", diskBalanceState=" + diskBalanceState +
|
||||
", bytesInAvgResource=" + bytesInAvgResource +
|
||||
", bytesInUtilization=" + bytesInUtilization +
|
||||
", bytesInBalanceState=" + bytesInBalanceState +
|
||||
", bytesOutAvgResource=" + bytesOutAvgResource +
|
||||
", bytesOutUtilization=" + bytesOutUtilization +
|
||||
", bytesOutBalanceState=" + bytesOutBalanceState +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
public class HostEnv {
|
||||
//BrokerId
|
||||
private int id;
|
||||
//机器IP
|
||||
private String host;
|
||||
//机架ID
|
||||
private String rackId;
|
||||
//CPU核数
|
||||
private int cpu;
|
||||
//磁盘总容量
|
||||
private double disk;
|
||||
//网卡容量
|
||||
private double network;
|
||||
|
||||
public int getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(int id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public String getRackId() {
|
||||
return rackId;
|
||||
}
|
||||
|
||||
public void setRackId(String rackId) {
|
||||
this.rackId = rackId;
|
||||
}
|
||||
|
||||
public int getCpu() {
|
||||
return cpu;
|
||||
}
|
||||
|
||||
public void setCpu(int cpu) {
|
||||
this.cpu = cpu;
|
||||
}
|
||||
|
||||
public double getDisk() {
|
||||
return disk;
|
||||
}
|
||||
|
||||
public void setDisk(double disk) {
|
||||
this.disk = disk;
|
||||
}
|
||||
|
||||
public double getNetwork() {
|
||||
return network;
|
||||
}
|
||||
|
||||
public void setNetwork(double network) {
|
||||
this.network = network;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HostEnv{" +
|
||||
"id=" + id +
|
||||
", host='" + host + '\'' +
|
||||
", rackId='" + rackId + '\'' +
|
||||
", cpu=" + cpu +
|
||||
", disk=" + disk +
|
||||
", network=" + network +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ExecutionProposal;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class OptimizerResult {
|
||||
private static final Logger logger = LoggerFactory.getLogger(OptimizerResult.class);
|
||||
private Set<ExecutionProposal> _proposals;
|
||||
private final BalanceParameter parameter;
|
||||
private Set<Broker> _balanceBrokersBefore;
|
||||
private Set<Broker> _balanceBrokersAfter;
|
||||
private final ClusterModel clusterModel;
|
||||
private final Map<TopicPartition, List<BalanceActionHistory>> balanceActionHistory;
|
||||
private final Map<String, BalanceThreshold> balanceThreshold;
|
||||
|
||||
public OptimizerResult(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
|
||||
this.clusterModel = clusterModel;
|
||||
balanceActionHistory = clusterModel.balanceActionHistory();
|
||||
parameter = optimizationOptions.parameter();
|
||||
double[] clusterAvgResource = clusterModel.avgOfUtilization();
|
||||
balanceThreshold = GoalUtils.getBalanceThreshold(parameter, clusterAvgResource);
|
||||
}
|
||||
|
||||
/**
|
||||
* 计划概览
|
||||
*/
|
||||
public BalanceOverview resultOverview() {
|
||||
BalanceOverview overview = new BalanceOverview();
|
||||
overview.setTopicBlacklist(parameter.getExcludedTopics());
|
||||
overview.setMoveReplicas(_proposals.size());
|
||||
overview.setNodeRange(parameter.getBalanceBrokers());
|
||||
overview.setRemoveNode(parameter.getOfflineBrokers());
|
||||
Map<Resource, Double> balanceThreshold = new HashMap<>();
|
||||
balanceThreshold.put(Resource.CPU, parameter.getCpuThreshold());
|
||||
balanceThreshold.put(Resource.DISK, parameter.getDiskThreshold());
|
||||
balanceThreshold.put(Resource.NW_IN, parameter.getNetworkInThreshold());
|
||||
balanceThreshold.put(Resource.NW_OUT, parameter.getNetworkOutThreshold());
|
||||
overview.setBalanceThreshold(balanceThreshold);
|
||||
Set<String> moveTopicsSet = _proposals.stream().map(j -> j.tp().topic()).collect(Collectors.toSet());
|
||||
String moveTopics = String.join(",", moveTopicsSet);
|
||||
overview.setMoveTopics(moveTopics);
|
||||
//Leader切换时不需要进行统计
|
||||
double totalMoveSize = _proposals.stream().filter(i -> Integer.max(i.replicasToAdd().size(), i.replicasToRemove().size()) != 0).mapToDouble(ExecutionProposal::partitionSize).sum();
|
||||
overview.setTotalMoveSize(totalMoveSize);
|
||||
return overview;
|
||||
}
|
||||
|
||||
/**
|
||||
* 计划明细
|
||||
*/
|
||||
public Map<Integer, BalanceDetailed> resultDetailed() {
|
||||
Map<Integer, BalanceDetailed> details = new HashMap<>();
|
||||
_balanceBrokersBefore.forEach(i -> {
|
||||
BalanceDetailed balanceDetailed = new BalanceDetailed();
|
||||
balanceDetailed.setBrokerId(i.id());
|
||||
balanceDetailed.setHost(i.host());
|
||||
balanceDetailed.setCurrentCPUUtilization(i.utilizationFor(Resource.CPU));
|
||||
balanceDetailed.setCurrentDiskUtilization(i.utilizationFor(Resource.DISK));
|
||||
balanceDetailed.setCurrentNetworkInUtilization(i.utilizationFor(Resource.NW_IN));
|
||||
balanceDetailed.setCurrentNetworkOutUtilization(i.utilizationFor(Resource.NW_OUT));
|
||||
details.put(i.id(), balanceDetailed);
|
||||
});
|
||||
Map<Integer, Double> totalAddReplicaCount = new HashMap<>();
|
||||
Map<Integer, Double> totalAddDataSize = new HashMap<>();
|
||||
Map<Integer, Double> totalRemoveReplicaCount = new HashMap<>();
|
||||
Map<Integer, Double> totalRemoveDataSize = new HashMap<>();
|
||||
_proposals.forEach(i -> {
|
||||
i.replicasToAdd().forEach((k, v) -> {
|
||||
totalAddReplicaCount.merge(k, v[0], Double::sum);
|
||||
totalAddDataSize.merge(k, v[1], Double::sum);
|
||||
});
|
||||
i.replicasToRemove().forEach((k, v) -> {
|
||||
totalRemoveReplicaCount.merge(k, v[0], Double::sum);
|
||||
totalRemoveDataSize.merge(k, v[1], Double::sum);
|
||||
});
|
||||
});
|
||||
_balanceBrokersAfter.forEach(i -> {
|
||||
BalanceDetailed balanceDetailed = details.get(i.id());
|
||||
balanceDetailed.setLastCPUUtilization(i.utilizationFor(Resource.CPU));
|
||||
balanceDetailed.setLastDiskUtilization(i.utilizationFor(Resource.DISK));
|
||||
balanceDetailed.setLastNetworkInUtilization(i.utilizationFor(Resource.NW_IN));
|
||||
balanceDetailed.setLastNetworkOutUtilization(i.utilizationFor(Resource.NW_OUT));
|
||||
balanceDetailed.setMoveInReplicas(totalAddReplicaCount.getOrDefault(i.id(), 0.0));
|
||||
balanceDetailed.setMoveOutReplicas(totalRemoveReplicaCount.getOrDefault(i.id(), 0.0));
|
||||
balanceDetailed.setMoveInDiskSize(totalAddDataSize.getOrDefault(i.id(), 0.0));
|
||||
balanceDetailed.setMoveOutDiskSize(totalRemoveDataSize.getOrDefault(i.id(), 0.0));
|
||||
for (String str : parameter.getGoals()) {
|
||||
BalanceThreshold threshold = balanceThreshold.get(str);
|
||||
if (!threshold.isInRange(i.utilizationFor(threshold.resource()))) {
|
||||
balanceDetailed.setBalanceState(-1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return details;
|
||||
}
|
||||
|
||||
/**
|
||||
* 计划任务
|
||||
*/
|
||||
public List<BalanceTask> resultTask() {
|
||||
List<BalanceTask> balanceTasks = new ArrayList<>();
|
||||
_proposals.forEach(proposal -> {
|
||||
BalanceTask task = new BalanceTask();
|
||||
task.setTopic(proposal.tp().topic());
|
||||
task.setPartition(proposal.tp().partition());
|
||||
List<Integer> replicas = proposal.newReplicas().stream().map(ReplicaPlacementInfo::brokerId).collect(Collectors.toList());
|
||||
task.setReplicas(replicas);
|
||||
balanceTasks.add(task);
|
||||
});
|
||||
return balanceTasks;
|
||||
}
|
||||
|
||||
public Map<TopicPartition, List<BalanceActionHistory>> resultBalanceActionHistory() {
|
||||
return Collections.unmodifiableMap(balanceActionHistory);
|
||||
}
|
||||
|
||||
public String resultJsonOverview() {
|
||||
try {
|
||||
return new ObjectMapper().writeValueAsString(resultOverview());
|
||||
} catch (Exception e) {
|
||||
logger.error("result overview json process error", e);
|
||||
}
|
||||
return "{}";
|
||||
}
|
||||
|
||||
public String resultJsonDetailed() {
|
||||
try {
|
||||
return new ObjectMapper().writeValueAsString(resultDetailed());
|
||||
} catch (Exception e) {
|
||||
logger.error("result detailed json process error", e);
|
||||
}
|
||||
return "{}";
|
||||
}
|
||||
|
||||
public String resultJsonTask() {
|
||||
try {
|
||||
Map<String, Object> reassign = new HashMap<>();
|
||||
reassign.put("partitions", resultTask());
|
||||
reassign.put("version", 1);
|
||||
return new ObjectMapper().writeValueAsString(reassign);
|
||||
} catch (Exception e) {
|
||||
logger.error("result task json process error", e);
|
||||
}
|
||||
return "{}";
|
||||
}
|
||||
|
||||
public List<TopicChangeHistory> resultTopicChangeHistory() {
|
||||
List<TopicChangeHistory> topicChangeHistoryList = new ArrayList<>();
|
||||
for (ExecutionProposal proposal : _proposals) {
|
||||
TopicChangeHistory changeHistory = new TopicChangeHistory();
|
||||
changeHistory.setTopic(proposal.tp().topic());
|
||||
changeHistory.setPartition(proposal.tp().partition());
|
||||
changeHistory.setOldLeader(proposal.oldLeader().brokerId());
|
||||
changeHistory.setNewLeader(proposal.newReplicas().get(0).brokerId());
|
||||
List<Integer> balanceBefore = proposal.oldReplicas().stream().map(ReplicaPlacementInfo::brokerId).collect(Collectors.toList());
|
||||
List<Integer> balanceAfter = proposal.newReplicas().stream().map(ReplicaPlacementInfo::brokerId).collect(Collectors.toList());
|
||||
changeHistory.setBalanceBefore(balanceBefore);
|
||||
changeHistory.setBalanceAfter(balanceAfter);
|
||||
topicChangeHistoryList.add(changeHistory);
|
||||
}
|
||||
return topicChangeHistoryList;
|
||||
}
|
||||
|
||||
public String resultJsonTopicChangeHistory() {
|
||||
try {
|
||||
return new ObjectMapper().writeValueAsString(resultTopicChangeHistory());
|
||||
} catch (Exception e) {
|
||||
logger.error("result balance topic change history json process error", e);
|
||||
}
|
||||
return "{}";
|
||||
}
|
||||
|
||||
public String resultJsonBalanceActionHistory() {
|
||||
try {
|
||||
return new ObjectMapper().writeValueAsString(balanceActionHistory);
|
||||
} catch (Exception e) {
|
||||
logger.error("result balance action history json process error", e);
|
||||
}
|
||||
return "{}";
|
||||
}
|
||||
|
||||
public void setBalanceBrokersFormBefore(Set<Broker> balanceBrokersBefore) {
|
||||
_balanceBrokersBefore = new HashSet<>();
|
||||
balanceBrokersBefore.forEach(i -> {
|
||||
Broker broker = new Broker(i.rack(), i.id(), i.host(), false, i.capacity());
|
||||
broker.load().addLoad(i.load());
|
||||
_balanceBrokersBefore.add(broker);
|
||||
});
|
||||
}
|
||||
|
||||
public void setBalanceBrokersFormAfter(Set<Broker> balanceBrokersAfter) {
|
||||
_balanceBrokersAfter = balanceBrokersAfter;
|
||||
}
|
||||
|
||||
public void setExecutionProposal(Set<ExecutionProposal> proposals) {
|
||||
_proposals = proposals;
|
||||
}
|
||||
|
||||
// test
|
||||
public ClusterModel clusterModel() {
|
||||
return clusterModel;
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.executor.common;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class TopicChangeHistory {
|
||||
//均衡Topic
|
||||
private String topic;
|
||||
//均衡分区
|
||||
private int partition;
|
||||
//旧Leader的BrokerID
|
||||
private int oldLeader;
|
||||
//均衡前副本分布
|
||||
private List<Integer> balanceBefore;
|
||||
//新Leader的BrokerID
|
||||
private int newLeader;
|
||||
//均衡后副本分布
|
||||
private List<Integer> balanceAfter;
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public int getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(int partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
public int getOldLeader() {
|
||||
return oldLeader;
|
||||
}
|
||||
|
||||
public void setOldLeader(int oldLeader) {
|
||||
this.oldLeader = oldLeader;
|
||||
}
|
||||
|
||||
public List<Integer> getBalanceBefore() {
|
||||
return balanceBefore;
|
||||
}
|
||||
|
||||
public void setBalanceBefore(List<Integer> balanceBefore) {
|
||||
this.balanceBefore = balanceBefore;
|
||||
}
|
||||
|
||||
public int getNewLeader() {
|
||||
return newLeader;
|
||||
}
|
||||
|
||||
public void setNewLeader(int newLeader) {
|
||||
this.newLeader = newLeader;
|
||||
}
|
||||
|
||||
public List<Integer> getBalanceAfter() {
|
||||
return balanceAfter;
|
||||
}
|
||||
|
||||
public void setBalanceAfter(List<Integer> balanceAfter) {
|
||||
this.balanceAfter = balanceAfter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TopicChangeHistory{" +
|
||||
"topic='" + topic + '\'' +
|
||||
", partition='" + partition + '\'' +
|
||||
", oldLeader=" + oldLeader +
|
||||
", balanceBefore=" + balanceBefore +
|
||||
", newLeader=" + newLeader +
|
||||
", balanceAfter=" + balanceAfter +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.metric;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/12
|
||||
*/
|
||||
public class Metric {
|
||||
private String topic;
|
||||
private int partition;
|
||||
private double cpu;
|
||||
private double bytesIn;
|
||||
private double bytesOut;
|
||||
private double disk;
|
||||
|
||||
public Metric() {
|
||||
|
||||
}
|
||||
|
||||
public Metric(String topic, int partition, double cpu, double bytesIn, double bytesOut, double disk) {
|
||||
this.topic = topic;
|
||||
this.partition = partition;
|
||||
this.cpu = cpu;
|
||||
this.bytesIn = bytesIn;
|
||||
this.bytesOut = bytesOut;
|
||||
this.disk = disk;
|
||||
}
|
||||
|
||||
public String topic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public int partition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public double cpu() {
|
||||
return cpu;
|
||||
}
|
||||
|
||||
public double bytesIn() {
|
||||
return bytesIn;
|
||||
}
|
||||
|
||||
public double bytesOut() {
|
||||
return bytesOut;
|
||||
}
|
||||
|
||||
public double disk() {
|
||||
return disk;
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.metric;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public interface MetricStore {
|
||||
Metrics getMetrics(String clusterName, int beforeSeconds);
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.metric;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Load;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public class Metrics {
|
||||
private final Map<TopicPartition, Metric> metricByTopicPartition;
|
||||
|
||||
public Metrics() {
|
||||
this.metricByTopicPartition = new HashMap<>();
|
||||
}
|
||||
|
||||
public void addMetrics(Metric metric) {
|
||||
TopicPartition topicPartition = new TopicPartition(metric.topic(), metric.partition());
|
||||
this.metricByTopicPartition.put(topicPartition, metric);
|
||||
}
|
||||
|
||||
public List<Metric> values() {
|
||||
return Collections.unmodifiableList(new ArrayList<>(this.metricByTopicPartition.values()));
|
||||
}
|
||||
|
||||
public Metric metric(TopicPartition topicPartition) {
|
||||
return this.metricByTopicPartition.get(topicPartition);
|
||||
}
|
||||
|
||||
public Load load(TopicPartition topicPartition) {
|
||||
Metric metric = this.metricByTopicPartition.get(topicPartition);
|
||||
if (metric == null) {
|
||||
return null;
|
||||
}
|
||||
Load load = new Load();
|
||||
load.setLoad(Resource.CPU, metric.cpu());
|
||||
load.setLoad(Resource.NW_IN, metric.bytesIn());
|
||||
load.setLoad(Resource.NW_OUT, metric.bytesOut());
|
||||
load.setLoad(Resource.DISK, metric.disk());
|
||||
|
||||
return load;
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.metric.elasticsearch;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.metric.Metric;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.metric.MetricStore;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.metric.Metrics;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.elasticsearch.client.Request;
|
||||
import org.elasticsearch.client.Response;
|
||||
import org.elasticsearch.client.RestClient;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public class ElasticsearchMetricStore implements MetricStore {
|
||||
private final Logger logger = LoggerFactory.getLogger(ElasticsearchMetricStore.class);
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private final String hosts;
|
||||
private final String indexPrefix;
|
||||
private final String format;
|
||||
|
||||
public ElasticsearchMetricStore(String hosts, String indexPrefix) {
|
||||
this(hosts, indexPrefix, "yyyy-MM-dd");
|
||||
}
|
||||
|
||||
public ElasticsearchMetricStore(String hosts, String indexPrefix, String format) {
|
||||
this.hosts = hosts;
|
||||
this.indexPrefix = indexPrefix;
|
||||
this.format = format;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Metrics getMetrics(String clusterName, int beforeSeconds) {
|
||||
Metrics metrics = new Metrics();
|
||||
try {
|
||||
String metricsQueryJson = IOUtils.resourceToString("/MetricsQuery.json", StandardCharsets.UTF_8);
|
||||
metricsQueryJson = metricsQueryJson.replaceAll("<var_before_time>", Integer.toString(beforeSeconds))
|
||||
.replaceAll("<var_cluster_name>", clusterName);
|
||||
try (RestClient restClient = RestClient.builder(toHttpHosts(this.hosts)).build()) {
|
||||
Request request = new Request(
|
||||
"GET",
|
||||
"/" + indices(beforeSeconds) + "/_search");
|
||||
request.setJsonEntity(metricsQueryJson);
|
||||
logger.debug("Es metrics query for cluster: {} request: {} dsl: {}", clusterName, request, metricsQueryJson);
|
||||
Response response = restClient.performRequest(request);
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
JsonNode rootNode = objectMapper.readTree(response.getEntity().getContent());
|
||||
JsonNode topics = rootNode.at("/aggregations/by_topic/buckets");
|
||||
for (JsonNode topic : topics) {
|
||||
String topicName = topic.path("key").asText();
|
||||
JsonNode partitions = topic.at("/by_partition/buckets");
|
||||
for (JsonNode partition : partitions) {
|
||||
int partitionId = partition.path("key").asInt();
|
||||
// double cpu = partition.at("/avg_cpu/value").asDouble();
|
||||
double cpu = 0D;
|
||||
double bytesIn = partition.at("/avg_bytes_in/value").asDouble();
|
||||
double bytesOut = partition.at("/avg_bytes_out/value").asDouble();
|
||||
double disk = partition.at("/lastest_disk/hits/hits/0/_source/metrics/LogSize").asDouble();
|
||||
// add
|
||||
metrics.addMetrics(new Metric(topicName, partitionId, cpu, bytesIn, bytesOut, disk));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("Cannot get metrics of cluster: " + clusterName, e);
|
||||
}
|
||||
logger.debug("Es metrics query for cluster: {} result count: {}", clusterName, metrics.values().size());
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private String indices(long beforeSeconds) {
|
||||
Set<String> indices = new TreeSet<>();
|
||||
DateFormat df = new SimpleDateFormat(this.format);
|
||||
long endTime = System.currentTimeMillis();
|
||||
long time = endTime - (beforeSeconds * 1000);
|
||||
while (time < endTime) {
|
||||
indices.add(this.indexPrefix + df.format(new Date(time)));
|
||||
time += 24 * 60 * 60 * 1000; // add 24h
|
||||
}
|
||||
indices.add(this.indexPrefix + df.format(new Date(endTime)));
|
||||
return String.join(",", indices);
|
||||
}
|
||||
|
||||
private static HttpHost[] toHttpHosts(String url) {
|
||||
String[] nodes = url.split(",");
|
||||
HttpHost[] hosts = new HttpHost[nodes.length];
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
String [] ipAndPort = nodes[i].split(":");
|
||||
hosts[i] = new HttpHost(ipAndPort[0], ipAndPort.length > 1 ? Integer.parseInt(ipAndPort[1]) : 9200);
|
||||
}
|
||||
return hosts;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public class Broker implements Comparable<Broker> {
|
||||
public static final Broker NONE = new Broker(new Rack("-1"), -1, "localhost", true, new Capacity());
|
||||
|
||||
private final Rack rack;
|
||||
private final int id;
|
||||
private final String host;
|
||||
private final boolean isOffline;
|
||||
|
||||
private final Set<Replica> replicas;
|
||||
private final Set<Replica> leaderReplicas;
|
||||
private final Map<String, Map<Integer, Replica>> topicReplicas;
|
||||
|
||||
private final Load load;
|
||||
|
||||
private final Capacity capacity;
|
||||
|
||||
public Broker(Rack rack, int id, String host, boolean isOffline, Capacity capacity) {
|
||||
this.rack = rack;
|
||||
this.id = id;
|
||||
this.host = host;
|
||||
this.isOffline = isOffline;
|
||||
this.replicas = new HashSet<>();
|
||||
this.leaderReplicas = new HashSet<>();
|
||||
this.topicReplicas = new HashMap<>();
|
||||
this.load = new Load();
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
public Rack rack() {
|
||||
return rack;
|
||||
}
|
||||
|
||||
public int id() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String host() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public boolean isOffline() {
|
||||
return isOffline;
|
||||
}
|
||||
|
||||
public Set<Replica> replicas() {
|
||||
return Collections.unmodifiableSet(this.replicas);
|
||||
}
|
||||
|
||||
public SortedSet<Replica> sortedReplicasFor(Resource resource, boolean reverse) {
|
||||
return sortedReplicasFor(null, resource, reverse);
|
||||
}
|
||||
|
||||
public SortedSet<Replica> sortedReplicasFor(Predicate<? super Replica> filter, Resource resource, boolean reverse) {
|
||||
Comparator<Replica> comparator =
|
||||
Comparator.<Replica>comparingDouble(r -> r.load().loadFor(resource))
|
||||
.thenComparingInt(Replica::hashCode);
|
||||
if (reverse)
|
||||
comparator = comparator.reversed();
|
||||
SortedSet<Replica> sortedReplicas = new TreeSet<>(comparator);
|
||||
if (filter == null) {
|
||||
sortedReplicas.addAll(this.replicas);
|
||||
} else {
|
||||
sortedReplicas.addAll(this.replicas.stream()
|
||||
.filter(filter).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
return sortedReplicas;
|
||||
}
|
||||
|
||||
public Set<Replica> leaderReplicas() {
|
||||
return Collections.unmodifiableSet(this.leaderReplicas);
|
||||
}
|
||||
|
||||
public Load load() {
|
||||
return load;
|
||||
}
|
||||
|
||||
public Capacity capacity() {
|
||||
return capacity;
|
||||
}
|
||||
|
||||
public double utilizationFor(Resource resource) {
|
||||
return this.load.loadFor(resource) / this.capacity.capacityFor(resource);
|
||||
}
|
||||
|
||||
public double expectedUtilizationAfterAdd(Resource resource, Load loadToChange) {
|
||||
return (this.load.loadFor(resource) + ((loadToChange == null) ? 0 : loadToChange.loadFor(resource)))
|
||||
/ this.capacity.capacityFor(resource);
|
||||
}
|
||||
|
||||
public double expectedUtilizationAfterRemove(Resource resource, Load loadToChange) {
|
||||
return (this.load.loadFor(resource) - ((loadToChange == null) ? 0 : loadToChange.loadFor(resource)))
|
||||
/ this.capacity.capacityFor(resource);
|
||||
}
|
||||
|
||||
public Replica replica(TopicPartition topicPartition) {
|
||||
Map<Integer, Replica> replicas = this.topicReplicas.get(topicPartition.topic());
|
||||
if (replicas == null) {
|
||||
return null;
|
||||
}
|
||||
return replicas.get(topicPartition.partition());
|
||||
}
|
||||
|
||||
void addReplica(Replica replica) {
|
||||
// Add replica to list of all replicas in the broker.
|
||||
if (this.replicas.contains(replica)) {
|
||||
throw new IllegalStateException(String.format("Broker %d already has replica %s", this.id,
|
||||
replica.topicPartition()));
|
||||
}
|
||||
this.replicas.add(replica);
|
||||
// Add topic replica.
|
||||
this.topicReplicas.computeIfAbsent(replica.topicPartition().topic(), t -> new HashMap<>())
|
||||
.put(replica.topicPartition().partition(), replica);
|
||||
|
||||
// Add leader replica.
|
||||
if (replica.isLeader()) {
|
||||
this.leaderReplicas.add(replica);
|
||||
}
|
||||
|
||||
// Add replica load to the broker load.
|
||||
this.load.addLoad(replica.load());
|
||||
}
|
||||
|
||||
Replica removeReplica(TopicPartition topicPartition) {
|
||||
Replica replica = replica(topicPartition);
|
||||
if (replica != null) {
|
||||
this.replicas.remove(replica);
|
||||
Map<Integer, Replica> replicas = this.topicReplicas.get(topicPartition.topic());
|
||||
if (replicas != null) {
|
||||
replicas.remove(topicPartition.partition());
|
||||
}
|
||||
if (replica.isLeader()) {
|
||||
this.leaderReplicas.remove(replica);
|
||||
}
|
||||
this.load.subtractLoad(replica.load());
|
||||
}
|
||||
return replica;
|
||||
}
|
||||
|
||||
Load makeFollower(TopicPartition topicPartition) {
|
||||
Replica replica = replica(topicPartition);
|
||||
Load leaderLoadDelta = replica.makeFollower();
|
||||
// Remove leadership load from load.
|
||||
this.load.subtractLoad(leaderLoadDelta);
|
||||
this.leaderReplicas.remove(replica);
|
||||
return leaderLoadDelta;
|
||||
}
|
||||
|
||||
void makeLeader(TopicPartition topicPartition, Load leaderLoadDelta) {
|
||||
Replica replica = replica(topicPartition);
|
||||
replica.makeLeader(leaderLoadDelta);
|
||||
// Add leadership load to load.
|
||||
this.load.addLoad(leaderLoadDelta);
|
||||
this.leaderReplicas.add(replica);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Broker broker = (Broker) o;
|
||||
return id == broker.id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Broker o) {
|
||||
return Integer.compare(id, o.id());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Broker{" +
|
||||
"id=" + id +
|
||||
", host='" + host + '\'' +
|
||||
", rack=" + rack.id() +
|
||||
", replicas=" + replicas +
|
||||
", leaderReplicas=" + leaderReplicas +
|
||||
", topicReplicas=" + topicReplicas +
|
||||
", load=" + load +
|
||||
", capacity=" + capacity +
|
||||
'}';
|
||||
}
|
||||
|
||||
public int numLeadersFor(String topicName) {
|
||||
return (int) replicasOfTopicInBroker(topicName).stream().filter(Replica::isLeader).count();
|
||||
}
|
||||
|
||||
public Set<String> topics() {
|
||||
return topicReplicas.keySet();
|
||||
}
|
||||
|
||||
public int numReplicasOfTopicInBroker(String topic) {
|
||||
Map<Integer, Replica> replicaMap = topicReplicas.get(topic);
|
||||
return replicaMap == null ? 0 : replicaMap.size();
|
||||
}
|
||||
|
||||
public Collection<Replica> replicasOfTopicInBroker(String topic) {
|
||||
Map<Integer, Replica> replicaMap = topicReplicas.get(topic);
|
||||
return replicaMap == null ? Collections.emptySet() : replicaMap.values();
|
||||
}
|
||||
|
||||
public Set<Replica> currentOfflineReplicas() {
|
||||
return replicas.stream().filter(Replica::isCurrentOffline).collect(Collectors.toSet());
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/9
|
||||
*/
|
||||
public class Capacity {
|
||||
private final double[] values;
|
||||
|
||||
public Capacity() {
|
||||
this.values = new double[Resource.values().length];
|
||||
}
|
||||
|
||||
public void setCapacity(Resource resource, double capacity) {
|
||||
this.values[resource.id()] = capacity;
|
||||
}
|
||||
|
||||
public double capacityFor(Resource resource) {
|
||||
return this.values[resource.id()];
|
||||
}
|
||||
|
||||
public void addCapacity(Capacity capacityToAdd) {
|
||||
for (Resource resource : Resource.values()) {
|
||||
this.setCapacity(resource, this.capacityFor(resource) + capacityToAdd.capacityFor(resource));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Capacity{" +
|
||||
"values=" + Arrays.toString(values) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceActionHistory;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public class ClusterModel {
|
||||
private final Map<String, Rack> racksById;
|
||||
private final Map<Integer, Broker> brokersById;
|
||||
private final Map<String, Map<TopicPartition, Partition>> partitionsByTopic;
|
||||
private Map<TopicPartition, List<BalanceActionHistory>> balanceActionHistory;
|
||||
|
||||
public ClusterModel() {
|
||||
this.racksById = new HashMap<>();
|
||||
this.brokersById = new HashMap<>();
|
||||
this.partitionsByTopic = new HashMap<>();
|
||||
this.balanceActionHistory = new HashMap<>();
|
||||
}
|
||||
|
||||
public Rack rack(String rackId) {
|
||||
return this.racksById.get(rackId);
|
||||
}
|
||||
|
||||
public Rack addRack(String rackId) {
|
||||
Rack rack = new Rack(rackId);
|
||||
this.racksById.putIfAbsent(rackId, rack);
|
||||
return this.racksById.get(rackId);
|
||||
}
|
||||
|
||||
public SortedSet<Broker> brokers() {
|
||||
return new TreeSet<>(this.brokersById.values());
|
||||
}
|
||||
|
||||
public Set<String> topics() {
|
||||
return this.partitionsByTopic.keySet();
|
||||
}
|
||||
|
||||
public SortedSet<Partition> topic(String name) {
|
||||
return new TreeSet<>(this.partitionsByTopic.get(name).values());
|
||||
}
|
||||
|
||||
public SortedSet<Broker> sortedBrokersFor(Resource resource, boolean reverse) {
|
||||
return sortedBrokersFor(null, resource, reverse);
|
||||
}
|
||||
|
||||
public SortedSet<Broker> sortedBrokersFor(Predicate<? super Broker> filter, Resource resource, boolean reverse) {
|
||||
Comparator<Broker> comparator =
|
||||
Comparator.<Broker>comparingDouble(b -> b.utilizationFor(resource))
|
||||
.thenComparingInt(Broker::id);
|
||||
if (reverse)
|
||||
comparator = comparator.reversed();
|
||||
SortedSet<Broker> sortedBrokers = new TreeSet<>(comparator);
|
||||
if (filter == null) {
|
||||
sortedBrokers.addAll(this.brokersById.values());
|
||||
} else {
|
||||
sortedBrokers.addAll(this.brokersById.values().stream()
|
||||
.filter(filter).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
return sortedBrokers;
|
||||
}
|
||||
|
||||
public Load load() {
|
||||
Load load = new Load();
|
||||
for (Broker broker : this.brokersById.values()) {
|
||||
load.addLoad(broker.load());
|
||||
}
|
||||
return load;
|
||||
}
|
||||
|
||||
public Capacity capacity() {
|
||||
Capacity capacity = new Capacity();
|
||||
for (Broker broker : this.brokersById.values()) {
|
||||
capacity.addCapacity(broker.capacity());
|
||||
}
|
||||
return capacity;
|
||||
}
|
||||
|
||||
public double utilizationFor(Resource resource) {
|
||||
return load().loadFor(resource) / capacity().capacityFor(resource);
|
||||
}
|
||||
|
||||
public double[] avgOfUtilization() {
|
||||
Load load = load();
|
||||
Capacity capacity = capacity();
|
||||
double[] unils = new double[Resource.values().length];
|
||||
for (Resource resource : Resource.values()) {
|
||||
unils[resource.id()] = load.loadFor(resource) / capacity.capacityFor(resource);
|
||||
}
|
||||
return unils;
|
||||
}
|
||||
|
||||
public Broker broker(int brokerId) {
|
||||
return this.brokersById.get(brokerId);
|
||||
}
|
||||
|
||||
public Broker addBroker(String rackId, int brokerId, String host, boolean isOffline, Capacity capacity) {
|
||||
Rack rack = rack(rackId);
|
||||
if (rack == null)
|
||||
throw new IllegalArgumentException("Rack: " + rackId + "is not exists.");
|
||||
Broker broker = new Broker(rack, brokerId, host, isOffline, capacity);
|
||||
rack.addBroker(broker);
|
||||
this.brokersById.put(brokerId, broker);
|
||||
return broker;
|
||||
}
|
||||
|
||||
public Replica addReplica(int brokerId, TopicPartition topicPartition, boolean isLeader, Load load) {
|
||||
return addReplica(brokerId, topicPartition, isLeader, false, load);
|
||||
}
|
||||
|
||||
public Replica addReplica(int brokerId, TopicPartition topicPartition, boolean isLeader, boolean isOffline, Load load) {
|
||||
Broker broker = broker(brokerId);
|
||||
if (broker == null) {
|
||||
throw new IllegalArgumentException("Broker: " + brokerId + "is not exists.");
|
||||
}
|
||||
|
||||
Replica replica = new Replica(broker, topicPartition, isLeader, isOffline);
|
||||
replica.setLoad(load);
|
||||
// add to broker
|
||||
broker.addReplica(replica);
|
||||
|
||||
Map<TopicPartition, Partition> partitions = this.partitionsByTopic
|
||||
.computeIfAbsent(topicPartition.topic(), k -> new HashMap<>());
|
||||
|
||||
Partition partition = partitions.computeIfAbsent(topicPartition, Partition::new);
|
||||
if (isLeader) {
|
||||
partition.addLeader(replica, 0);
|
||||
} else {
|
||||
partition.addFollower(replica, partition.replicas().size());
|
||||
}
|
||||
|
||||
return replica;
|
||||
}
|
||||
|
||||
public Replica removeReplica(int brokerId, TopicPartition topicPartition) {
|
||||
Broker broker = broker(brokerId);
|
||||
return broker.removeReplica(topicPartition);
|
||||
}
|
||||
|
||||
public void relocateLeadership(String goal, String actionType, TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) {
|
||||
relocateLeadership(topicPartition, sourceBrokerId, destinationBrokerId);
|
||||
addBalanceActionHistory(goal, actionType, topicPartition, sourceBrokerId, destinationBrokerId);
|
||||
}
|
||||
|
||||
public void relocateLeadership(TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) {
|
||||
Broker sourceBroker = broker(sourceBrokerId);
|
||||
Replica sourceReplica = sourceBroker.replica(topicPartition);
|
||||
if (!sourceReplica.isLeader()) {
|
||||
throw new IllegalArgumentException("Cannot relocate leadership of partition " + topicPartition + "from broker "
|
||||
+ sourceBrokerId + " to broker " + destinationBrokerId
|
||||
+ " because the source replica isn't leader.");
|
||||
}
|
||||
Broker destinationBroker = broker(destinationBrokerId);
|
||||
Replica destinationReplica = destinationBroker.replica(topicPartition);
|
||||
if (destinationReplica.isLeader()) {
|
||||
throw new IllegalArgumentException("Cannot relocate leadership of partition " + topicPartition + "from broker "
|
||||
+ sourceBrokerId + " to broker " + destinationBrokerId
|
||||
+ " because the destination replica is a leader.");
|
||||
}
|
||||
Load leaderLoadDelta = sourceBroker.makeFollower(topicPartition);
|
||||
destinationBroker.makeLeader(topicPartition, leaderLoadDelta);
|
||||
|
||||
Partition partition = this.partitionsByTopic.get(topicPartition.topic()).get(topicPartition);
|
||||
partition.relocateLeadership(destinationReplica);
|
||||
}
|
||||
|
||||
public void relocateReplica(String goal, String actionType, TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) {
|
||||
relocateReplica(topicPartition, sourceBrokerId, destinationBrokerId);
|
||||
addBalanceActionHistory(goal, actionType, topicPartition, sourceBrokerId, destinationBrokerId);
|
||||
}
|
||||
|
||||
public void relocateReplica(TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) {
|
||||
Replica replica = removeReplica(sourceBrokerId, topicPartition);
|
||||
if (replica == null) {
|
||||
throw new IllegalArgumentException("Replica is not in the cluster.");
|
||||
}
|
||||
Broker destinationBroker = broker(destinationBrokerId);
|
||||
replica.setBroker(destinationBroker);
|
||||
destinationBroker.addReplica(replica);
|
||||
}
|
||||
|
||||
private void addBalanceActionHistory(String goal, String actionType, TopicPartition topicPartition, int sourceBrokerId, int destinationBrokerId) {
|
||||
BalanceActionHistory history = new BalanceActionHistory();
|
||||
history.setActionType(actionType);
|
||||
history.setGoal(goal);
|
||||
history.setTopic(topicPartition.topic());
|
||||
history.setPartition(topicPartition.partition());
|
||||
history.setSourceBrokerId(sourceBrokerId);
|
||||
history.setDestinationBrokerId(destinationBrokerId);
|
||||
this.balanceActionHistory.computeIfAbsent(topicPartition, k -> new ArrayList<>()).add(history);
|
||||
}
|
||||
|
||||
public Map<String, Integer> numLeadersPerTopic(Set<String> topics) {
|
||||
Map<String, Integer> leaderCountByTopicNames = new HashMap<>();
|
||||
topics.forEach(topic -> leaderCountByTopicNames.put(topic, partitionsByTopic.get(topic).size()));
|
||||
return leaderCountByTopicNames;
|
||||
}
|
||||
|
||||
public Map<TopicPartition, List<ReplicaPlacementInfo>> getReplicaDistribution() {
|
||||
Map<TopicPartition, List<ReplicaPlacementInfo>> replicaDistribution = new HashMap<>();
|
||||
for (Map<TopicPartition, Partition> tp : partitionsByTopic.values()) {
|
||||
tp.values().forEach(i -> {
|
||||
i.replicas().forEach(j -> replicaDistribution.computeIfAbsent(j.topicPartition(), k -> new ArrayList<>())
|
||||
.add(new ReplicaPlacementInfo(j.broker().id(), "")));
|
||||
});
|
||||
}
|
||||
return replicaDistribution;
|
||||
}
|
||||
|
||||
public Replica partition(TopicPartition tp) {
|
||||
return partitionsByTopic.get(tp.topic()).get(tp).leader();
|
||||
}
|
||||
|
||||
public Map<TopicPartition, ReplicaPlacementInfo> getLeaderDistribution() {
|
||||
Map<TopicPartition, ReplicaPlacementInfo> leaderDistribution = new HashMap<>();
|
||||
for (Broker broker : brokersById.values()) {
|
||||
broker.leaderReplicas().forEach(i -> leaderDistribution.put(i.topicPartition(), new ReplicaPlacementInfo(broker.id(), "")));
|
||||
}
|
||||
return leaderDistribution;
|
||||
}
|
||||
|
||||
public int numTopicReplicas(String topic) {
|
||||
return partitionsByTopic.get(topic).size();
|
||||
}
|
||||
|
||||
public Map<TopicPartition, List<BalanceActionHistory>> balanceActionHistory() {
|
||||
return this.balanceActionHistory;
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/9
|
||||
*/
|
||||
public class Load {
|
||||
private final double[] values;
|
||||
|
||||
public Load() {
|
||||
this.values = new double[Resource.values().length];
|
||||
}
|
||||
|
||||
public void setLoad(Resource resource, double load) {
|
||||
this.values[resource.id()] = load;
|
||||
}
|
||||
|
||||
public double loadFor(Resource resource) {
|
||||
return this.values[resource.id()];
|
||||
}
|
||||
|
||||
public void addLoad(Load loadToAdd) {
|
||||
for (Resource resource : Resource.values()) {
|
||||
this.setLoad(resource, this.loadFor(resource) + loadToAdd.loadFor(resource));
|
||||
}
|
||||
}
|
||||
|
||||
public void subtractLoad(Load loadToSubtract) {
|
||||
for (Resource resource : Resource.values()) {
|
||||
this.setLoad(resource, this.loadFor(resource) - loadToSubtract.loadFor(resource));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Load{" +
|
||||
"values=" + Arrays.toString(values) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/11
|
||||
*/
|
||||
public class Partition implements Comparable<Partition> {
|
||||
private final TopicPartition topicPartition;
|
||||
private final List<Replica> replicas;
|
||||
|
||||
public Partition(TopicPartition topicPartition) {
|
||||
this.topicPartition = topicPartition;
|
||||
this.replicas = new ArrayList<>();
|
||||
}
|
||||
|
||||
public TopicPartition topicPartition() {
|
||||
return topicPartition;
|
||||
}
|
||||
|
||||
public List<Replica> replicas() {
|
||||
return replicas;
|
||||
}
|
||||
|
||||
public Broker originalLeaderBroker() {
|
||||
return replicas.stream().filter(r -> r.original().isLeader())
|
||||
.findFirst().orElseThrow(IllegalStateException::new).broker();
|
||||
}
|
||||
|
||||
public Replica leader() {
|
||||
return replicas.stream()
|
||||
.filter(Replica::isLeader)
|
||||
.findFirst()
|
||||
.orElseThrow(() ->
|
||||
new IllegalArgumentException("Not found leader of partition " + topicPartition)
|
||||
);
|
||||
}
|
||||
|
||||
public Replica leaderOrNull() {
|
||||
return replicas.stream()
|
||||
.filter(Replica::isLeader)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
public List<Replica> followers() {
|
||||
return replicas.stream()
|
||||
.filter(r -> !r.isLeader())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
Replica replica(long brokerId) {
|
||||
return replicas.stream()
|
||||
.filter(r -> r.broker().id() == brokerId)
|
||||
.findFirst()
|
||||
.orElseThrow(() ->
|
||||
new IllegalArgumentException("Requested replica " + brokerId + " is not a replica of partition " + topicPartition)
|
||||
);
|
||||
}
|
||||
|
||||
public boolean isLeaderChanged() {
|
||||
// return originalLeaderBroker() != this.leader().broker();
|
||||
return replicas.stream().anyMatch(Replica::isLeaderChanged);
|
||||
}
|
||||
|
||||
public boolean isChanged() {
|
||||
return replicas.stream().anyMatch(Replica::isChanged);
|
||||
}
|
||||
|
||||
void addLeader(Replica leader, int index) {
|
||||
if (leaderOrNull() != null) {
|
||||
throw new IllegalArgumentException(String.format("Partition %s already has a leader replica %s. Cannot "
|
||||
+ "add a new leader replica %s", this.topicPartition, leaderOrNull(), leader));
|
||||
}
|
||||
if (!leader.isLeader()) {
|
||||
throw new IllegalArgumentException("Inconsistent leadership information. Trying to set " + leader.broker()
|
||||
+ " as the leader for partition " + this.topicPartition + " while the replica is not marked "
|
||||
+ "as a leader.");
|
||||
}
|
||||
this.replicas.add(index, leader);
|
||||
}
|
||||
|
||||
void addFollower(Replica follower, int index) {
|
||||
if (follower.isLeader()) {
|
||||
throw new IllegalArgumentException("Inconsistent leadership information. Trying to add follower replica "
|
||||
+ follower + " while it is a leader.");
|
||||
}
|
||||
if (!follower.topicPartition().equals(this.topicPartition)) {
|
||||
throw new IllegalArgumentException("Inconsistent topic partition. Trying to add follower replica " + follower
|
||||
+ " to partition " + this.topicPartition + ".");
|
||||
}
|
||||
this.replicas.add(index, follower);
|
||||
}
|
||||
|
||||
void relocateLeadership(Replica newLeader) {
|
||||
if (!newLeader.isLeader()) {
|
||||
throw new IllegalArgumentException("Inconsistent leadership information. Trying to set " + newLeader.broker()
|
||||
+ " as the leader for partition " + this.topicPartition + " while the replica is not marked "
|
||||
+ "as a leader.");
|
||||
}
|
||||
int leaderPos = this.replicas.indexOf(newLeader);
|
||||
swapReplicaPositions(0, leaderPos);
|
||||
}
|
||||
|
||||
void swapReplicaPositions(int index1, int index2) {
|
||||
Replica replica1 = this.replicas.get(index1);
|
||||
Replica replica2 = this.replicas.get(index2);
|
||||
|
||||
this.replicas.set(index2, replica1);
|
||||
this.replicas.set(index1, replica2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Partition partition = (Partition) o;
|
||||
return topicPartition.equals(partition.topicPartition);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(topicPartition);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Partition{" +
|
||||
"topicPartition=" + topicPartition +
|
||||
", replicas=" + replicas +
|
||||
", originalLeaderBroker=" + originalLeaderBroker().id() +
|
||||
", leader=" + leaderOrNull() +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(Partition o) {
|
||||
return Integer.compare(topicPartition.partition(), o.topicPartition.partition());
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/9
|
||||
*/
|
||||
public class Rack {
|
||||
private final String id;
|
||||
private final SortedSet<Broker> brokers;
|
||||
|
||||
public Rack(String id) {
|
||||
this.id = id;
|
||||
this.brokers = new TreeSet<>();
|
||||
}
|
||||
|
||||
public String id() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public SortedSet<Broker> brokers() {
|
||||
return Collections.unmodifiableSortedSet(this.brokers);
|
||||
}
|
||||
|
||||
public Load load() {
|
||||
Load load = new Load();
|
||||
for (Broker broker : this.brokers) {
|
||||
load.addLoad(broker.load());
|
||||
}
|
||||
return load;
|
||||
}
|
||||
|
||||
public List<Replica> replicas() {
|
||||
List<Replica> replicas = new ArrayList<>();
|
||||
for (Broker broker : this.brokers) {
|
||||
replicas.addAll(broker.replicas());
|
||||
}
|
||||
return replicas;
|
||||
}
|
||||
|
||||
Broker addBroker(Broker broker) {
|
||||
this.brokers.add(broker);
|
||||
return broker;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Rack rack = (Rack) o;
|
||||
return Objects.equals(id, rack.id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Rack{" +
|
||||
"id='" + id + '\'' +
|
||||
", brokers=" + brokers +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public class Replica {
|
||||
private final Load load;
|
||||
private final Replica original;
|
||||
private final TopicPartition topicPartition;
|
||||
private Broker broker;
|
||||
private boolean isLeader;
|
||||
private boolean isOffline;
|
||||
|
||||
public Replica(Broker broker, TopicPartition topicPartition, boolean isLeader, boolean isOffline) {
|
||||
this(broker, topicPartition, isLeader, isOffline, false);
|
||||
}
|
||||
|
||||
private Replica(Broker broker, TopicPartition topicPartition, boolean isLeader, boolean isOffline, boolean isOriginal) {
|
||||
if (isOriginal) {
|
||||
this.original = null;
|
||||
} else {
|
||||
this.original = new Replica(broker, topicPartition, isLeader, isOffline, true);
|
||||
}
|
||||
this.load = new Load();
|
||||
this.topicPartition = topicPartition;
|
||||
this.broker = broker;
|
||||
this.isLeader = isLeader;
|
||||
this.isOffline = isOffline;
|
||||
}
|
||||
|
||||
public TopicPartition topicPartition() {
|
||||
return topicPartition;
|
||||
}
|
||||
|
||||
public Replica original() {
|
||||
return original;
|
||||
}
|
||||
|
||||
public Broker broker() {
|
||||
return broker;
|
||||
}
|
||||
|
||||
public void setBroker(Broker broker) {
|
||||
checkOriginal();
|
||||
this.broker = broker;
|
||||
}
|
||||
|
||||
public boolean isLeader() {
|
||||
return isLeader;
|
||||
}
|
||||
|
||||
public Load load() {
|
||||
return load;
|
||||
}
|
||||
|
||||
void setLoad(Load load) {
|
||||
checkOriginal();
|
||||
this.load.addLoad(load);
|
||||
}
|
||||
|
||||
Load makeFollower() {
|
||||
checkOriginal();
|
||||
this.isLeader = false;
|
||||
// TODO cpu recal
|
||||
Load leaderLoadDelta = new Load();
|
||||
leaderLoadDelta.setLoad(Resource.NW_OUT, this.load.loadFor(Resource.NW_OUT));
|
||||
this.load.subtractLoad(leaderLoadDelta);
|
||||
return leaderLoadDelta;
|
||||
}
|
||||
|
||||
void makeLeader(Load leaderLoadDelta) {
|
||||
checkOriginal();
|
||||
this.isLeader = true;
|
||||
this.load.addLoad(leaderLoadDelta);
|
||||
}
|
||||
|
||||
public boolean isLeaderChanged() {
|
||||
checkOriginal();
|
||||
return this.original.isLeader != this.isLeader;
|
||||
}
|
||||
|
||||
public boolean isChanged() {
|
||||
checkOriginal();
|
||||
return this.original.broker != this.broker || this.original.isLeader != this.isLeader;
|
||||
}
|
||||
|
||||
private void checkOriginal() {
|
||||
if (this.original == null) {
|
||||
throw new IllegalStateException("This is a original replica, this operation is not supported.");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
checkOriginal();
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Replica replica = (Replica) o;
|
||||
return topicPartition.equals(replica.topicPartition) && this.original.broker.equals(replica.original.broker);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
checkOriginal();
|
||||
return Objects.hash(topicPartition, this.original.broker);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
checkOriginal();
|
||||
return "Replica{" +
|
||||
"topicPartition=" + topicPartition +
|
||||
", originalBroker=" + this.original.broker.id() +
|
||||
", broker=" + broker.id() +
|
||||
", originalIsLeader=" + this.original.isLeader +
|
||||
", isLeader=" + isLeader +
|
||||
", load=" + load +
|
||||
'}';
|
||||
}
|
||||
//todo:副本状态，待考虑
|
||||
public boolean isCurrentOffline() {
|
||||
return isOffline;
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public class ReplicaPlacementInfo {
|
||||
private final int _brokerId;
|
||||
private final String _logdir;
|
||||
|
||||
public ReplicaPlacementInfo(int brokerId, String logdir) {
|
||||
_brokerId = brokerId;
|
||||
_logdir = logdir;
|
||||
}
|
||||
|
||||
public ReplicaPlacementInfo(Integer brokerId) {
|
||||
this(brokerId, null);
|
||||
}
|
||||
|
||||
public Integer brokerId() {
|
||||
return _brokerId;
|
||||
}
|
||||
|
||||
public String logdir() {
|
||||
return _logdir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (!(o instanceof ReplicaPlacementInfo)) {
|
||||
return false;
|
||||
}
|
||||
ReplicaPlacementInfo info = (ReplicaPlacementInfo) o;
|
||||
return _brokerId == info._brokerId && Objects.equals(_logdir, info._logdir);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(_brokerId, _logdir);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
if (_logdir == null) {
|
||||
return String.format("{Broker: %d}", _brokerId);
|
||||
} else {
|
||||
return String.format("{Broker: %d, Logdir: %s}", _brokerId, _logdir);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/10
|
||||
*/
|
||||
public enum Resource {
|
||||
CPU("cpu", 0),
|
||||
NW_IN("bytesIn", 1),
|
||||
NW_OUT("bytesOut", 2),
|
||||
DISK("disk", 3);
|
||||
|
||||
private final String resource;
|
||||
private final int id;
|
||||
|
||||
Resource(String resource, int id) {
|
||||
this.resource = resource;
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String resource() {
|
||||
return this.resource;
|
||||
}
|
||||
|
||||
public int id() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.model;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.metric.MetricStore;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.metric.Metrics;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.metric.elasticsearch.ElasticsearchMetricStore;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.utils.MetadataUtils;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.Cluster;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/12
|
||||
*/
|
||||
public class Supplier {
|
||||
public static Map<String, String> subConfig(Map<String, String> config, String prefix, boolean stripPrefix) {
|
||||
return config.entrySet().stream()
|
||||
.filter(e -> e.getKey().startsWith(prefix))
|
||||
.collect(Collectors.toMap(e -> stripPrefix ? e.getKey().substring(prefix.length()) : e.getKey(),
|
||||
Map.Entry::getValue));
|
||||
}
|
||||
|
||||
public static ClusterModel load(String clusterName, int beforeSeconds, String kafkaBootstrapServer, String esUrls, String esIndexPrefix, Map<Integer, Capacity> capacitiesById, Set<String> ignoredTopics) {
|
||||
Properties kafkaProperties = new Properties();
|
||||
kafkaProperties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBootstrapServer);
|
||||
return load(clusterName, beforeSeconds, kafkaProperties, esUrls, esIndexPrefix, capacitiesById, ignoredTopics);
|
||||
}
|
||||
|
||||
public static ClusterModel load(String clusterName, int beforeSeconds, Properties kafkaProperties, String esUrls, String esIndexPrefix, Map<Integer, Capacity> capacitiesById, Set<String> ignoredTopics) {
|
||||
MetricStore store = new ElasticsearchMetricStore(esUrls, esIndexPrefix);
|
||||
Metrics metrics = store.getMetrics(clusterName, beforeSeconds);
|
||||
return load(kafkaProperties, capacitiesById, metrics, ignoredTopics);
|
||||
}
|
||||
|
||||
public static ClusterModel load(Properties kafkaProperties, Map<Integer, Capacity> capacitiesById, Metrics metrics, Set<String> ignoredTopics) {
|
||||
ClusterModel model = new ClusterModel();
|
||||
Cluster cluster = MetadataUtils.metadata(kafkaProperties);
|
||||
|
||||
// nodes
|
||||
for (Node node: cluster.nodes()) {
|
||||
addBroker(node, false, model, capacitiesById);
|
||||
}
|
||||
|
||||
// replicas
|
||||
cluster.topics()
|
||||
.stream()
|
||||
.filter(topic -> !ignoredTopics.contains(topic))
|
||||
.forEach(topic -> {
|
||||
List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
|
||||
for (PartitionInfo partition : partitions) {
|
||||
// TODO fix ignore no partition leader
|
||||
if (partition.leader() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
TopicPartition topicPartition = new TopicPartition(partition.topic(), partition.partition());
|
||||
Load leaderLoad = metrics.load(topicPartition);
|
||||
if (leaderLoad == null) {
|
||||
if (partition.leader() == null) {
|
||||
// set empty load
|
||||
leaderLoad = new Load();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Cannot get leader load of topic partiton: " + topicPartition);
|
||||
}
|
||||
}
|
||||
|
||||
// leader nw out + follower nw out
|
||||
leaderLoad.setLoad(Resource.NW_OUT,
|
||||
leaderLoad.loadFor(Resource.NW_OUT) +
|
||||
leaderLoad.loadFor(Resource.NW_IN) * (partition.replicas().length - 1));
|
||||
|
||||
Load followerLoad = new Load();
|
||||
followerLoad.addLoad(leaderLoad);
|
||||
followerLoad.setLoad(Resource.NW_OUT, 0);
|
||||
List<Node> offlineReplicas = Arrays.asList(partition.offlineReplicas());
|
||||
for (Node n : partition.replicas()) {
|
||||
boolean isLeader = partition.leader() != null && partition.leader().equals(n);
|
||||
boolean isOffline = offlineReplicas.contains(n);
|
||||
if (isOffline) {
|
||||
if (model.broker(n.id()) == null) {
|
||||
// add offline broker
|
||||
addBroker(n, true, model, capacitiesById);
|
||||
}
|
||||
}
|
||||
model.addReplica(n.id(), topicPartition, isLeader, isOffline, isLeader ? leaderLoad : followerLoad);
|
||||
}
|
||||
}
|
||||
});
|
||||
return model;
|
||||
}
|
||||
|
||||
private static String rack(Node node) {
|
||||
return (node.rack() == null || "".equals(node.rack())) ? node.host() : node.rack();
|
||||
}
|
||||
|
||||
private static void addBroker(Node node, boolean isOffline, ClusterModel model, Map<Integer, Capacity> capacitiesById) {
|
||||
// rack
|
||||
Rack rack = model.addRack(rack(node));
|
||||
// broker
|
||||
Capacity capacity = capacitiesById.get(node.id());
|
||||
if (capacity == null)
|
||||
throw new IllegalArgumentException("Cannot get capacity of node: " + node);
|
||||
|
||||
model.addBroker(rack.id(), node.id(), node.host(), isOffline, capacity);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
public enum ActionAcceptance {
|
||||
ACCEPT, REJECT;
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
public enum ActionType {
|
||||
REPLICA_MOVEMENT("REPLICA"),
|
||||
LEADERSHIP_MOVEMENT("LEADER");
|
||||
// REPLICA_SWAP("SWAP");
|
||||
|
||||
private final String _balancingAction;
|
||||
|
||||
ActionType(String balancingAction) {
|
||||
_balancingAction = balancingAction;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return _balancingAction;
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Replica;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance.ACCEPT;
|
||||
|
||||
public class AnalyzerUtils {
|
||||
|
||||
public static Set<String> getSplitTopics(String value) {
|
||||
if (StringUtils.isBlank(value)) {
|
||||
return new HashSet<>();
|
||||
}
|
||||
String[] arr = value.split(",");
|
||||
return Arrays.stream(arr).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public static Set<Integer> getSplitBrokers(String value) {
|
||||
if (StringUtils.isBlank(value)) {
|
||||
return new HashSet<>();
|
||||
}
|
||||
String[] arr = value.split(",");
|
||||
return Arrays.stream(arr).map(Integer::valueOf).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public static Set<ExecutionProposal> getDiff(Map<TopicPartition, List<ReplicaPlacementInfo>> initialReplicaDistribution,
|
||||
Map<TopicPartition, ReplicaPlacementInfo> initialLeaderDistribution,
|
||||
ClusterModel optimizedClusterModel) {
|
||||
Map<TopicPartition, List<ReplicaPlacementInfo>> finalReplicaDistribution = optimizedClusterModel.getReplicaDistribution();
|
||||
if (!initialReplicaDistribution.keySet().equals(finalReplicaDistribution.keySet())) {
|
||||
throw new IllegalArgumentException("diff distributions with different partitions.");
|
||||
}
|
||||
Set<ExecutionProposal> diff = new HashSet<>();
|
||||
for (Map.Entry<TopicPartition, List<ReplicaPlacementInfo>> entry : initialReplicaDistribution.entrySet()) {
|
||||
TopicPartition tp = entry.getKey();
|
||||
List<ReplicaPlacementInfo> initialReplicas = entry.getValue();
|
||||
List<ReplicaPlacementInfo> finalReplicas = finalReplicaDistribution.get(tp);
|
||||
Replica finalLeader = optimizedClusterModel.partition(tp);
|
||||
ReplicaPlacementInfo finalLeaderPlacementInfo = new ReplicaPlacementInfo(finalLeader.broker().id(), "");
|
||||
if (finalReplicas.equals(initialReplicas) && initialLeaderDistribution.get(tp).equals(finalLeaderPlacementInfo)) {
|
||||
continue;
|
||||
}
|
||||
if (!finalLeaderPlacementInfo.equals(finalReplicas.get(0))) {
|
||||
int leaderPos = finalReplicas.indexOf(finalLeaderPlacementInfo);
|
||||
finalReplicas.set(leaderPos, finalReplicas.get(0));
|
||||
finalReplicas.set(0, finalLeaderPlacementInfo);
|
||||
}
|
||||
double partitionSize = optimizedClusterModel.partition(tp).load().loadFor(Resource.DISK);
|
||||
diff.add(new ExecutionProposal(tp, partitionSize, initialLeaderDistribution.get(tp), initialReplicas, finalReplicas));
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
|
||||
public static ActionAcceptance isProposalAcceptableForOptimizedGoals(Set<Goal> optimizedGoals,
|
||||
BalancingAction proposal,
|
||||
ClusterModel clusterModel) {
|
||||
for (Goal optimizedGoal : optimizedGoals) {
|
||||
ActionAcceptance actionAcceptance = optimizedGoal.actionAcceptance(proposal, clusterModel);
|
||||
if (actionAcceptance != ACCEPT) {
|
||||
return actionAcceptance;
|
||||
}
|
||||
}
|
||||
return ACCEPT;
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
public class BalancingAction {
|
||||
private final TopicPartition _tp;
|
||||
private final Integer _sourceBrokerId;
|
||||
private final Integer _destinationBrokerId;
|
||||
private final ActionType _actionType;
|
||||
|
||||
public BalancingAction(TopicPartition tp,
|
||||
Integer sourceBrokerId,
|
||||
Integer destinationBrokerId,
|
||||
ActionType actionType) {
|
||||
_tp = tp;
|
||||
_sourceBrokerId = sourceBrokerId;
|
||||
_destinationBrokerId = destinationBrokerId;
|
||||
_actionType = actionType;
|
||||
}
|
||||
|
||||
public Integer sourceBrokerId() {
|
||||
return _sourceBrokerId;
|
||||
}
|
||||
|
||||
public Integer destinationBrokerId() {
|
||||
return _destinationBrokerId;
|
||||
}
|
||||
|
||||
public ActionType balancingAction() {
|
||||
return _actionType;
|
||||
}
|
||||
|
||||
public TopicPartition topicPartition() {
|
||||
return _tp;
|
||||
}
|
||||
|
||||
public String topic() {
|
||||
return _tp.topic();
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ExecutionProposal {
|
||||
|
||||
private final TopicPartition _tp;
|
||||
private final double _partitionSize;
|
||||
private final ReplicaPlacementInfo _oldLeader;
|
||||
private final List<ReplicaPlacementInfo> _oldReplicas;
|
||||
private final List<ReplicaPlacementInfo> _newReplicas;
|
||||
private final Set<ReplicaPlacementInfo> _replicasToAdd;
|
||||
private final Set<ReplicaPlacementInfo> _replicasToRemove;
|
||||
|
||||
public ExecutionProposal(TopicPartition tp,
|
||||
double partitionSize,
|
||||
ReplicaPlacementInfo oldLeader,
|
||||
List<ReplicaPlacementInfo> oldReplicas,
|
||||
List<ReplicaPlacementInfo> newReplicas) {
|
||||
_tp = tp;
|
||||
_partitionSize = partitionSize;
|
||||
_oldLeader = oldLeader;
|
||||
_oldReplicas = oldReplicas == null ? Collections.emptyList() : oldReplicas;
|
||||
_newReplicas = newReplicas;
|
||||
Set<Integer> newBrokerList = _newReplicas.stream().mapToInt(ReplicaPlacementInfo::brokerId).boxed().collect(Collectors.toSet());
|
||||
Set<Integer> oldBrokerList = _oldReplicas.stream().mapToInt(ReplicaPlacementInfo::brokerId).boxed().collect(Collectors.toSet());
|
||||
_replicasToAdd = _newReplicas.stream().filter(r -> !oldBrokerList.contains(r.brokerId())).collect(Collectors.toSet());
|
||||
_replicasToRemove = _oldReplicas.stream().filter(r -> !newBrokerList.contains(r.brokerId())).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public TopicPartition tp() {
|
||||
return _tp;
|
||||
}
|
||||
|
||||
public double partitionSize() {
|
||||
return _partitionSize;
|
||||
}
|
||||
|
||||
public ReplicaPlacementInfo oldLeader() {
|
||||
return _oldLeader;
|
||||
}
|
||||
|
||||
public List<ReplicaPlacementInfo> oldReplicas() {
|
||||
return _oldReplicas;
|
||||
}
|
||||
|
||||
public List<ReplicaPlacementInfo> newReplicas() {
|
||||
return _newReplicas;
|
||||
}
|
||||
|
||||
public Map<Integer, Double[]> replicasToAdd() {
|
||||
Map<Integer, Double[]> addData = new HashMap<>();
|
||||
_replicasToAdd.forEach(i -> {
|
||||
Double[] total = {1d, _partitionSize};
|
||||
addData.put(i.brokerId(), total);
|
||||
});
|
||||
return Collections.unmodifiableMap(addData);
|
||||
}
|
||||
|
||||
public Map<Integer, Double[]> replicasToRemove() {
|
||||
Map<Integer, Double[]> removeData = new HashMap<>();
|
||||
_replicasToRemove.forEach(i -> {
|
||||
Double[] total = {1d, _partitionSize};
|
||||
removeData.put(i.brokerId(), total);
|
||||
});
|
||||
return Collections.unmodifiableMap(removeData);
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ReplicaPlacementInfo;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/4/29
|
||||
*/
|
||||
public class GoalOptimizer {
|
||||
private static final Logger logger = LoggerFactory.getLogger(GoalOptimizer.class);
|
||||
|
||||
public OptimizerResult optimizations(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
|
||||
Set<Goal> optimizedGoals = new HashSet<>();
|
||||
OptimizerResult optimizerResult = new OptimizerResult(clusterModel, optimizationOptions);
|
||||
optimizerResult.setBalanceBrokersFormBefore(clusterModel.brokers());
|
||||
Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = clusterModel.getReplicaDistribution();
|
||||
Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = clusterModel.getLeaderDistribution();
|
||||
try {
|
||||
Map<String, Goal> goalMap = new HashMap<>();
|
||||
ServiceLoader<Goal> serviceLoader = ServiceLoader.load(Goal.class);
|
||||
for (Goal goal : serviceLoader) {
|
||||
goalMap.put(goal.name(), goal);
|
||||
}
|
||||
for (String g : optimizationOptions.goals()) {
|
||||
Goal goal = goalMap.get(g);
|
||||
if (goal != null) {
|
||||
logger.info("Start {} balancing", goal.name());
|
||||
goal.optimize(clusterModel, optimizedGoals, optimizationOptions);
|
||||
optimizedGoals.add(goal);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Cluster balancing goal error", e);
|
||||
}
|
||||
Set<ExecutionProposal> proposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, clusterModel);
|
||||
optimizerResult.setBalanceBrokersFormAfter(clusterModel.brokers());
|
||||
optimizerResult.setExecutionProposal(proposals);
|
||||
return optimizerResult;
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class OptimizationOptions {
|
||||
private final Set<String> _excludedTopics;
|
||||
private final Set<Integer> _offlineBrokers;
|
||||
private final Set<Integer> _balanceBrokers;
|
||||
private final Map<Resource, Double> _resourceBalancePercentage;
|
||||
private final List<String> _goals;
|
||||
private final BalanceParameter _parameter;
|
||||
|
||||
public OptimizationOptions(BalanceParameter parameter) {
|
||||
_parameter = parameter;
|
||||
_goals = parameter.getGoals();
|
||||
_excludedTopics = AnalyzerUtils.getSplitTopics(parameter.getExcludedTopics());
|
||||
_offlineBrokers = AnalyzerUtils.getSplitBrokers(parameter.getOfflineBrokers());
|
||||
_balanceBrokers = AnalyzerUtils.getSplitBrokers(parameter.getBalanceBrokers());
|
||||
_resourceBalancePercentage = new HashMap<>();
|
||||
_resourceBalancePercentage.put(Resource.CPU, parameter.getCpuThreshold());
|
||||
_resourceBalancePercentage.put(Resource.DISK, parameter.getDiskThreshold());
|
||||
_resourceBalancePercentage.put(Resource.NW_IN, parameter.getNetworkInThreshold());
|
||||
_resourceBalancePercentage.put(Resource.NW_OUT, parameter.getNetworkOutThreshold());
|
||||
}
|
||||
|
||||
public Set<String> excludedTopics() {
|
||||
return Collections.unmodifiableSet(_excludedTopics);
|
||||
}
|
||||
|
||||
public Set<Integer> offlineBrokers() {
|
||||
return Collections.unmodifiableSet(_offlineBrokers);
|
||||
}
|
||||
|
||||
public Set<Integer> balanceBrokers() {
|
||||
return Collections.unmodifiableSet(_balanceBrokers);
|
||||
}
|
||||
|
||||
public double resourceBalancePercentageFor(Resource resource) {
|
||||
return _resourceBalancePercentage.get(resource);
|
||||
}
|
||||
|
||||
public List<String> goals() {
|
||||
return Collections.unmodifiableList(_goals);
|
||||
}
|
||||
|
||||
public double topicReplicaThreshold() {
|
||||
return _parameter.getTopicReplicaThreshold();
|
||||
}
|
||||
|
||||
public BalanceParameter parameter() {
|
||||
return _parameter;
|
||||
}
|
||||
|
||||
public double topicLeaderThreshold() {
|
||||
return _parameter.getTopicLeaderThreshold();
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Replica;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.*;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public abstract class AbstractGoal implements Goal {
|
||||
|
||||
/**
|
||||
* 均衡算法逻辑处理
|
||||
*/
|
||||
protected abstract void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions);
|
||||
|
||||
/**
|
||||
* 集群列表中的所有Broker循环执行均衡算法
|
||||
*/
|
||||
@Override
|
||||
public void optimize(ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
|
||||
initGoalState(clusterModel, optimizationOptions);
|
||||
SortedSet<Broker> brokenBrokers = clusterModel.brokers().stream()
|
||||
.filter(b -> optimizationOptions.balanceBrokers().isEmpty()
|
||||
|| optimizationOptions.balanceBrokers().contains(b.id()))
|
||||
.collect(Collectors.toCollection(TreeSet::new));
|
||||
|
||||
// SortedSet<Broker> brokenBrokers = clusterModel.brokers();
|
||||
|
||||
for (Broker broker : brokenBrokers) {
|
||||
rebalanceForBroker(broker, clusterModel, optimizedGoals, optimizationOptions);
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions);
|
||||
|
||||
/**
|
||||
* 根据已经计算完的均衡副本、候选目标Broker、执行类型来
|
||||
* 执行不同的集群模型数据更改操作
|
||||
*/
|
||||
protected Broker maybeApplyBalancingAction(ClusterModel clusterModel,
|
||||
Replica replica,
|
||||
Collection<Broker> candidateBrokers,
|
||||
ActionType action,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
List<Broker> eligibleBrokers = eligibleBrokers(replica, candidateBrokers, action, optimizationOptions);
|
||||
for (Broker broker : eligibleBrokers) {
|
||||
BalancingAction proposal = new BalancingAction(replica.topicPartition(), replica.broker().id(), broker.id(), action);
|
||||
//均衡的副本如果存在当前的Broker上则进行下次Broker
|
||||
if (!legitMove(replica, broker, action)) {
|
||||
continue;
|
||||
}
|
||||
//均衡条件已经满足进行下次Broker
|
||||
if (!selfSatisfied(clusterModel, proposal)) {
|
||||
continue;
|
||||
}
|
||||
//判断当前均衡操作是否与其他目标冲突,如果冲突则禁止均衡操作
|
||||
ActionAcceptance acceptance = AnalyzerUtils.isProposalAcceptableForOptimizedGoals(optimizedGoals, proposal, clusterModel);
|
||||
if (acceptance == ActionAcceptance.ACCEPT) {
|
||||
if (action == ActionType.LEADERSHIP_MOVEMENT) {
|
||||
clusterModel.relocateLeadership(name(), action.toString(), replica.topicPartition(), replica.broker().id(), broker.id());
|
||||
} else if (action == ActionType.REPLICA_MOVEMENT) {
|
||||
clusterModel.relocateReplica(name(), action.toString(), replica.topicPartition(), replica.broker().id(), broker.id());
|
||||
}
|
||||
return broker;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 副本操作合法性判断:
|
||||
* 1.副本迁移，目的broker不包含移动副本
|
||||
* 2.Leader切换，目的broker需要包含切换副本
|
||||
*/
|
||||
private static boolean legitMove(Replica replica,
|
||||
Broker destinationBroker, ActionType actionType) {
|
||||
switch (actionType) {
|
||||
case REPLICA_MOVEMENT:
|
||||
return destinationBroker.replica(replica.topicPartition()) == null;
|
||||
case LEADERSHIP_MOVEMENT:
|
||||
return replica.isLeader() && destinationBroker.replica(replica.topicPartition()) != null;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action);
|
||||
|
||||
/**
|
||||
* 候选Broker列表筛选过滤
|
||||
*/
|
||||
public static List<Broker> eligibleBrokers(Replica replica,
|
||||
Collection<Broker> candidates,
|
||||
ActionType action,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
List<Broker> eligibleBrokers = new ArrayList<>(candidates);
|
||||
filterOutBrokersExcludedForLeadership(eligibleBrokers, optimizationOptions, replica, action);
|
||||
filterOutBrokersExcludedForReplicaMove(eligibleBrokers, optimizationOptions, action);
|
||||
return eligibleBrokers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Leader切换，从候选的Broker列表中排除掉excludedBroker
|
||||
*/
|
||||
public static void filterOutBrokersExcludedForLeadership(List<Broker> eligibleBrokers,
|
||||
OptimizationOptions optimizationOptions,
|
||||
Replica replica,
|
||||
ActionType action) {
|
||||
Set<Integer> excludedBrokers = optimizationOptions.offlineBrokers();
|
||||
if (!excludedBrokers.isEmpty() && (action == ActionType.LEADERSHIP_MOVEMENT || replica.isLeader())) {
|
||||
eligibleBrokers.removeIf(broker -> excludedBrokers.contains(broker.id()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 副本迁移，从候选的Broker列表中排除掉excludedBroker
|
||||
*/
|
||||
public static void filterOutBrokersExcludedForReplicaMove(List<Broker> eligibleBrokers,
|
||||
OptimizationOptions optimizationOptions,
|
||||
ActionType action) {
|
||||
Set<Integer> excludedBrokers = optimizationOptions.offlineBrokers();
|
||||
if (!excludedBrokers.isEmpty() && action == ActionType.REPLICA_MOVEMENT) {
|
||||
eligibleBrokers.removeIf(broker -> excludedBrokers.contains(broker.id()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/24
|
||||
*/
|
||||
public class DiskDistributionGoal extends ResourceDistributionGoal {
|
||||
|
||||
@Override
|
||||
protected Resource resource() {
|
||||
return Resource.DISK;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return DiskDistributionGoal.class.getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
|
||||
// Leadership movement won't cause disk utilization change.
|
||||
return action.balancingAction() == ActionType.LEADERSHIP_MOVEMENT ? ActionAcceptance.ACCEPT : super.actionAcceptance(action, clusterModel);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
public interface Goal {
|
||||
|
||||
void optimize(ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions);
|
||||
|
||||
String name();
|
||||
|
||||
ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel);
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/20
|
||||
*/
|
||||
public class NetworkInboundDistributionGoal extends ResourceDistributionGoal {
|
||||
|
||||
@Override
|
||||
protected Resource resource() {
|
||||
return Resource.NW_IN;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return NetworkInboundDistributionGoal.class.getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
|
||||
// Leadership movement won't cause inbound network utilization change.
|
||||
return action.balancingAction() == ActionType.LEADERSHIP_MOVEMENT ? ActionAcceptance.ACCEPT : super.actionAcceptance(action, clusterModel);
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Resource;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/24
|
||||
*/
|
||||
public class NetworkOutboundDistributionGoal extends ResourceDistributionGoal {
|
||||
|
||||
@Override
|
||||
protected Resource resource() {
|
||||
return Resource.NW_OUT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return NetworkOutboundDistributionGoal.class.getSimpleName();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.*;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/20
|
||||
*/
|
||||
public abstract class ResourceDistributionGoal extends AbstractGoal {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ResourceDistributionGoal.class);
|
||||
private double balanceUpperThreshold;
|
||||
private double balanceLowerThreshold;
|
||||
|
||||
@Override
|
||||
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
|
||||
double avgUtilization = clusterModel.utilizationFor(resource());
|
||||
double balancePercentage = optimizationOptions.resourceBalancePercentageFor(resource());
|
||||
this.balanceUpperThreshold = avgUtilization * (1 + balancePercentage);
|
||||
this.balanceLowerThreshold = avgUtilization * (1 - balancePercentage);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void rebalanceForBroker(Broker broker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
double utilization = broker.utilizationFor(resource());
|
||||
|
||||
boolean requireLessLoad = utilization > this.balanceUpperThreshold;
|
||||
boolean requireMoreLoad = utilization < this.balanceLowerThreshold;
|
||||
if (!requireMoreLoad && !requireLessLoad) {
|
||||
return;
|
||||
}
|
||||
|
||||
// First try leadership movement
|
||||
if (resource() == Resource.NW_OUT || resource() == Resource.CPU) {
|
||||
if (requireLessLoad && rebalanceByMovingLoadOut(broker, clusterModel, optimizedGoals,
|
||||
ActionType.LEADERSHIP_MOVEMENT, optimizationOptions)) {
|
||||
logger.debug("Successfully balanced {} for broker {} by moving out leaders.", resource(), broker.id());
|
||||
requireLessLoad = false;
|
||||
}
|
||||
if (requireMoreLoad && rebalanceByMovingLoadIn(broker, clusterModel, optimizedGoals,
|
||||
ActionType.LEADERSHIP_MOVEMENT, optimizationOptions)) {
|
||||
logger.debug("Successfully balanced {} for broker {} by moving in leaders.", resource(), broker.id());
|
||||
requireMoreLoad = false;
|
||||
}
|
||||
}
|
||||
|
||||
boolean balanced = true;
|
||||
if (requireLessLoad) {
|
||||
if (!rebalanceByMovingLoadOut(broker, clusterModel, optimizedGoals,
|
||||
ActionType.REPLICA_MOVEMENT, optimizationOptions)) {
|
||||
balanced = rebalanceBySwappingLoadOut(broker, clusterModel, optimizedGoals, optimizationOptions);
|
||||
}
|
||||
} else if (requireMoreLoad) {
|
||||
if (!rebalanceByMovingLoadIn(broker, clusterModel, optimizedGoals,
|
||||
ActionType.REPLICA_MOVEMENT, optimizationOptions)) {
|
||||
balanced = rebalanceBySwappingLoadIn(broker, clusterModel, optimizedGoals, optimizationOptions);
|
||||
}
|
||||
}
|
||||
if (balanced) {
|
||||
logger.debug("Successfully balanced {} for broker {} by moving leaders and replicas.", resource(), broker.id());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean rebalanceByMovingLoadOut(Broker broker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
ActionType actionType,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
|
||||
SortedSet<Broker> candidateBrokers = sortedCandidateBrokersUnderThreshold(clusterModel, this.balanceUpperThreshold, optimizationOptions, broker, false);
|
||||
SortedSet<Replica> replicasToMove = sortedCandidateReplicas(broker, actionType, optimizationOptions, true);
|
||||
|
||||
for (Replica replica : replicasToMove) {
|
||||
Broker acceptedBroker = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, actionType, optimizedGoals, optimizationOptions);
|
||||
|
||||
if (acceptedBroker != null) {
|
||||
if (broker.utilizationFor(resource()) < this.balanceUpperThreshold) {
|
||||
return true;
|
||||
}
|
||||
// Remove and reinsert the broker so the order is correct.
|
||||
// candidateBrokers.remove(acceptedBroker);
|
||||
candidateBrokers.removeIf(b -> b.id() == acceptedBroker.id());
|
||||
if (acceptedBroker.utilizationFor(resource()) < this.balanceUpperThreshold) {
|
||||
candidateBrokers.add(acceptedBroker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean rebalanceByMovingLoadIn(Broker broker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
ActionType actionType,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
SortedSet<Broker> candidateBrokers = sortedCandidateBrokersOverThreshold(clusterModel, this.balanceLowerThreshold, optimizationOptions, broker, true);
|
||||
Iterator<Broker> candidateBrokersIt = candidateBrokers.iterator();
|
||||
Broker nextCandidateBroker = null;
|
||||
while (true) {
|
||||
Broker candidateBroker;
|
||||
if (nextCandidateBroker != null) {
|
||||
candidateBroker = nextCandidateBroker;
|
||||
nextCandidateBroker = null;
|
||||
} else if (candidateBrokersIt.hasNext()) {
|
||||
candidateBroker = candidateBrokersIt.next();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
SortedSet<Replica> replicasToMove = sortedCandidateReplicas(candidateBroker, actionType, optimizationOptions, true);
|
||||
|
||||
for (Replica replica : replicasToMove) {
|
||||
Broker acceptedBroker = maybeApplyBalancingAction(clusterModel, replica, Collections.singletonList(broker), actionType, optimizedGoals, optimizationOptions);
|
||||
if (acceptedBroker != null) {
|
||||
if (broker.utilizationFor(resource()) > this.balanceLowerThreshold) {
|
||||
return true;
|
||||
}
|
||||
if (candidateBrokersIt.hasNext() || nextCandidateBroker != null) {
|
||||
if (nextCandidateBroker == null) {
|
||||
nextCandidateBroker = candidateBrokersIt.next();
|
||||
}
|
||||
if (candidateBroker.utilizationFor(resource()) < nextCandidateBroker.utilizationFor(resource())) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean rebalanceBySwappingLoadOut(Broker broker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean rebalanceBySwappingLoadIn(Broker broker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
return false;
|
||||
}
|
||||
|
||||
private SortedSet<Broker> sortedCandidateBrokersUnderThreshold(ClusterModel clusterModel,
|
||||
double utilizationThreshold,
|
||||
OptimizationOptions optimizationOptions,
|
||||
Broker excludedBroker,
|
||||
boolean reverse) {
|
||||
return clusterModel.sortedBrokersFor(
|
||||
b -> b.utilizationFor(resource()) < utilizationThreshold
|
||||
&& !excludedBroker.equals(b)
|
||||
// filter brokers
|
||||
&& (optimizationOptions.balanceBrokers().isEmpty() || optimizationOptions.balanceBrokers().contains(b.id()))
|
||||
, resource(), reverse);
|
||||
}
|
||||
|
||||
private SortedSet<Broker> sortedCandidateBrokersOverThreshold(ClusterModel clusterModel,
|
||||
double utilizationThreshold,
|
||||
OptimizationOptions optimizationOptions,
|
||||
Broker excludedBroker,
|
||||
boolean reverse) {
|
||||
return clusterModel.sortedBrokersFor(
|
||||
b -> b.utilizationFor(resource()) > utilizationThreshold
|
||||
&& !excludedBroker.equals(b)
|
||||
// filter brokers
|
||||
&& (optimizationOptions.balanceBrokers().isEmpty() || optimizationOptions.balanceBrokers().contains(b.id()))
|
||||
, resource(), reverse);
|
||||
}
|
||||
|
||||
private SortedSet<Replica> sortedCandidateReplicas(Broker broker,
|
||||
ActionType actionType,
|
||||
OptimizationOptions optimizationOptions,
|
||||
boolean reverse) {
|
||||
return broker.sortedReplicasFor(
|
||||
// exclude topic
|
||||
r -> !optimizationOptions.excludedTopics().contains(r.topicPartition().topic())
|
||||
&& r.load().loadFor(resource()) > 0.0
|
||||
// LEADERSHIP_MOVEMENT or NW_OUT is require leader replica
|
||||
&& (actionType != ActionType.LEADERSHIP_MOVEMENT && resource() != Resource.NW_OUT || r.isLeader())
|
||||
, resource(), reverse);
|
||||
}
|
||||
|
||||
protected abstract Resource resource();
|
||||
|
||||
@Override
|
||||
protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) {
|
||||
Broker destinationBroker = clusterModel.broker(action.destinationBrokerId());
|
||||
Broker sourceBroker = clusterModel.broker(action.sourceBrokerId());
|
||||
Replica sourceReplica = sourceBroker.replica(action.topicPartition());
|
||||
|
||||
Load loadToChange;
|
||||
if (action.balancingAction() == ActionType.LEADERSHIP_MOVEMENT) {
|
||||
Replica destinationReplica = destinationBroker.replica(action.topicPartition());
|
||||
Load delta = new Load();
|
||||
delta.addLoad(sourceReplica.load());
|
||||
delta.subtractLoad(destinationReplica.load());
|
||||
loadToChange = delta;
|
||||
} else {
|
||||
loadToChange = sourceReplica.load();
|
||||
}
|
||||
double sourceUtilization = sourceBroker.expectedUtilizationAfterRemove(resource(), loadToChange);
|
||||
double destinationUtilization = destinationBroker.expectedUtilizationAfterAdd(resource(), loadToChange);
|
||||
|
||||
return sourceUtilization >= this.balanceLowerThreshold && destinationUtilization <= this.balanceUpperThreshold;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
|
||||
return this.selfSatisfied(clusterModel, action) ? ActionAcceptance.ACCEPT : ActionAcceptance.REJECT;
|
||||
}
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Replica;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance.ACCEPT;
|
||||
import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance.REJECT;
|
||||
import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType.REPLICA_MOVEMENT;
|
||||
import static com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType.LEADERSHIP_MOVEMENT;
|
||||
|
||||
public class TopicLeadersDistributionGoal extends AbstractGoal {
|
||||
private static final Logger logger = LoggerFactory.getLogger(TopicLeadersDistributionGoal.class);
|
||||
private Map<String, Integer> _mustHaveTopicMinLeadersPerBroker;
|
||||
|
||||
/**
|
||||
* 执行Topic Leader均衡
|
||||
*/
|
||||
@Override
|
||||
protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
|
||||
moveAwayOfflineReplicas(broker, clusterModel, optimizedGoals, optimizationOptions);
|
||||
if (_mustHaveTopicMinLeadersPerBroker.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (optimizationOptions.offlineBrokers().contains(broker.id())) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (String topicName : _mustHaveTopicMinLeadersPerBroker.keySet()) {
|
||||
maybeMoveLeaderOfTopicToBroker(topicName, broker, clusterModel, optimizedGoals, optimizationOptions);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化均衡条件:
|
||||
* 1.排除不需要的Broker、Topic
|
||||
* 2.计算每个Topic在集群中所有Broker的平均分布数量
|
||||
*/
|
||||
@Override
|
||||
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
|
||||
_mustHaveTopicMinLeadersPerBroker = new HashMap<>();
|
||||
Set<String> excludedTopics = optimizationOptions.excludedTopics();
|
||||
Set<Integer> excludedBrokers = optimizationOptions.offlineBrokers();
|
||||
Set<String> mustHaveTopicLeadersPerBroker = GoalUtils.getNotExcludeTopics(clusterModel, excludedTopics);
|
||||
Map<String, Integer> numLeadersByTopicNames = clusterModel.numLeadersPerTopic(mustHaveTopicLeadersPerBroker);
|
||||
Set<Broker> allBrokers = GoalUtils.getNotExcludeBrokers(clusterModel, excludedBrokers);
|
||||
for (String topicName : mustHaveTopicLeadersPerBroker) {
|
||||
int topicNumLeaders = numLeadersByTopicNames.get(topicName);
|
||||
int avgLeaders = allBrokers.size() == 0 ? 0 : (int) Math.ceil(topicNumLeaders / (double) allBrokers.size() * (1 + optimizationOptions.topicLeaderThreshold()));
|
||||
_mustHaveTopicMinLeadersPerBroker.put(topicName, avgLeaders);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 已满足均衡条件判断:
|
||||
* 1.待操作Broker的副本已下线
|
||||
* 2.待操作Broker上Topic Leader数量大于Topic平均分布数量
|
||||
*/
|
||||
@Override
|
||||
protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) {
|
||||
Broker sourceBroker = clusterModel.broker(action.sourceBrokerId());
|
||||
Replica replicaToBeMoved = sourceBroker.replica(action.topicPartition());
|
||||
if (replicaToBeMoved.broker().replica(action.topicPartition()).isCurrentOffline()) {
|
||||
return action.balancingAction() == REPLICA_MOVEMENT;
|
||||
}
|
||||
String topicName = replicaToBeMoved.topicPartition().topic();
|
||||
return sourceBroker.numLeadersFor(topicName) > minTopicLeadersPerBroker(topicName);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取Topic在每台Broker上的最小Leader数
|
||||
*/
|
||||
private int minTopicLeadersPerBroker(String topicName) {
|
||||
return _mustHaveTopicMinLeadersPerBroker.get(topicName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return TopicLeadersDistributionGoal.class.getSimpleName();
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断Topic Leader均衡动作是否可以执行
|
||||
*/
|
||||
@Override
|
||||
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
|
||||
if (_mustHaveTopicMinLeadersPerBroker.containsKey(action.topic())) {
|
||||
return ACCEPT;
|
||||
}
|
||||
switch (action.balancingAction()) {
|
||||
case LEADERSHIP_MOVEMENT:
|
||||
case REPLICA_MOVEMENT:
|
||||
Replica replicaToBeRemoved = clusterModel.broker(action.sourceBrokerId()).replica(action.topicPartition());
|
||||
return doesLeaderRemoveViolateOptimizedGoal(replicaToBeRemoved) ? REJECT : ACCEPT;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported balancing action " + action.balancingAction() + " is provided.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据指定的副本判断是否可以执行均衡动作
|
||||
*/
|
||||
private boolean doesLeaderRemoveViolateOptimizedGoal(Replica replicaToBeRemoved) {
|
||||
if (!replicaToBeRemoved.isLeader()) {
|
||||
return false;
|
||||
}
|
||||
String topic = replicaToBeRemoved.topicPartition().topic();
|
||||
if (!_mustHaveTopicMinLeadersPerBroker.containsKey(topic)) {
|
||||
return false;
|
||||
}
|
||||
int topicLeaderCountOnSourceBroker = replicaToBeRemoved.broker().numLeadersFor(topic);
|
||||
return topicLeaderCountOnSourceBroker <= minTopicLeadersPerBroker(topic);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行具体的均衡逻辑:
|
||||
* 先通过Leader切换如果还不满足条件则进行副本迁移
|
||||
*/
|
||||
private void maybeMoveLeaderOfTopicToBroker(String topicName,
|
||||
Broker broker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
int topicLeaderCount = broker.numLeadersFor(topicName);
|
||||
//判断Topic在当前Broker上的Leader数量是否超过最小Leader分布
|
||||
if (topicLeaderCount >= minTopicLeadersPerBroker(topicName)) {
|
||||
return;
|
||||
}
|
||||
//获取Topic在当前Broker上的所有follower副本
|
||||
List<Replica> followerReplicas = broker.replicas().stream().filter(i -> !i.isLeader() && i.topicPartition().topic().equals(topicName)).collect(Collectors.toList());
|
||||
for (Replica followerReplica : followerReplicas) {
|
||||
//根据follower副本信息从集群中获取对应的Leader副本
|
||||
Replica leader = clusterModel.partition(followerReplica.topicPartition());
|
||||
//如果Leader副本所在Broker的Topic Leader数量超过最小Leader分布则进行Leader切换
|
||||
if (leader.broker().numLeadersFor(topicName) > minTopicLeadersPerBroker(topicName)) {
|
||||
if (maybeApplyBalancingAction(clusterModel, leader, Collections.singleton(broker),
|
||||
LEADERSHIP_MOVEMENT, optimizedGoals, optimizationOptions) != null) {
|
||||
topicLeaderCount++;
|
||||
//Topic在当前Broker的Leader分布大于等于最小Leader分布则结束均衡
|
||||
if (topicLeaderCount >= minTopicLeadersPerBroker(topicName)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//根据Topic获取需要Leader数量大于最小Leader分布待迁移的Broker列表
|
||||
PriorityQueue<Broker> brokersWithExcessiveLeaderToMove = getBrokersWithExcessiveLeaderToMove(topicName, clusterModel);
|
||||
while (!brokersWithExcessiveLeaderToMove.isEmpty()) {
|
||||
Broker brokerWithExcessiveLeaderToMove = brokersWithExcessiveLeaderToMove.poll();
|
||||
List<Replica> leadersOfTopic = brokerWithExcessiveLeaderToMove.leaderReplicas().stream()
|
||||
.filter(i -> i.topicPartition().topic().equals(topicName)).collect(Collectors.toList());
|
||||
boolean leaderMoved = false;
|
||||
int leaderMoveCount = leadersOfTopic.size();
|
||||
for (Replica leaderOfTopic : leadersOfTopic) {
|
||||
Broker destinationBroker = maybeApplyBalancingAction(clusterModel, leaderOfTopic, Collections.singleton(broker),
|
||||
REPLICA_MOVEMENT, optimizedGoals, optimizationOptions);
|
||||
if (destinationBroker != null) {
|
||||
leaderMoved = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (leaderMoved) {
|
||||
//当前Topic Leader数量在满足最小Leader分布后则结束均衡
|
||||
topicLeaderCount++;
|
||||
if (topicLeaderCount >= minTopicLeadersPerBroker(topicName)) {
|
||||
return;
|
||||
}
|
||||
//分布过多的Broker在进行副本迁移后Topic Leader依然大于最小Leader分布则继续迁移
|
||||
leaderMoveCount--;
|
||||
if (leaderMoveCount > minTopicLeadersPerBroker(topicName)) {
|
||||
brokersWithExcessiveLeaderToMove.add(brokerWithExcessiveLeaderToMove);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据指定的TopicName,筛选出集群内超过该TopicName Leader平均分布数量的所有Broker并且降序排列
|
||||
*/
|
||||
private PriorityQueue<Broker> getBrokersWithExcessiveLeaderToMove(String topicName, ClusterModel clusterModel) {
|
||||
PriorityQueue<Broker> brokersWithExcessiveLeaderToMove = new PriorityQueue<>((broker1, broker2) -> {
|
||||
int broker1LeaderCount = broker1.numLeadersFor(topicName);
|
||||
int broker2LeaderCount = broker2.numLeadersFor(topicName);
|
||||
int leaderCountCompareResult = Integer.compare(broker2LeaderCount, broker1LeaderCount);
|
||||
return leaderCountCompareResult == 0 ? Integer.compare(broker1.id(), broker2.id()) : leaderCountCompareResult;
|
||||
});
|
||||
clusterModel.brokers().stream().filter(broker -> broker.numLeadersFor(topicName) > minTopicLeadersPerBroker(topicName))
|
||||
.forEach(brokersWithExcessiveLeaderToMove::add);
|
||||
return brokersWithExcessiveLeaderToMove;
|
||||
}
|
||||
|
||||
/**
|
||||
* 下线副本优先处理迁移
|
||||
*/
|
||||
private void moveAwayOfflineReplicas(Broker srcBroker,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
if (srcBroker.currentOfflineReplicas().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
SortedSet<Broker> eligibleBrokersToMoveOfflineReplicasTo = new TreeSet<>(
|
||||
Comparator.comparingInt((Broker broker) -> broker.replicas().size()).thenComparingInt(Broker::id));
|
||||
Set<Replica> offlineReplicas = new HashSet<>(srcBroker.currentOfflineReplicas());
|
||||
for (Replica offlineReplica : offlineReplicas) {
|
||||
if (maybeApplyBalancingAction(clusterModel, offlineReplica, eligibleBrokersToMoveOfflineReplicasTo,
|
||||
REPLICA_MOVEMENT, optimizedGoals, optimizationOptions) == null) {
|
||||
logger.error(String.format("[%s] offline replica %s from broker %d (has %d replicas) move error", name(),
|
||||
offlineReplica, srcBroker.id(), srcBroker.replicas().size()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,287 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Replica;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionAcceptance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.ActionType;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.BalancingAction;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.OptimizationOptions;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.utils.GoalUtils;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class TopicReplicaDistributionGoal extends AbstractGoal {
|
||||
private final Map<String, Integer> _balanceUpperLimitByTopic;
|
||||
private final Map<String, Integer> _balanceLowerLimitByTopic;
|
||||
private Set<Broker> _brokersAllowedReplicaMove;
|
||||
private final Map<String, Double> _avgTopicReplicasOnBroker;
|
||||
|
||||
public TopicReplicaDistributionGoal() {
|
||||
_balanceUpperLimitByTopic = new HashMap<>();
|
||||
_balanceLowerLimitByTopic = new HashMap<>();
|
||||
_avgTopicReplicasOnBroker = new HashMap<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
|
||||
for (String topic : broker.topics()) {
|
||||
if (isTopicExcludedFromRebalance(topic)) {
|
||||
continue;
|
||||
}
|
||||
Collection<Replica> replicas = broker.replicasOfTopicInBroker(topic);
|
||||
int numTopicReplicas = replicas.size();
|
||||
boolean isExcludedForReplicaMove = isExcludedForReplicaMove(broker);
|
||||
int numOfflineTopicReplicas = GoalUtils.retainCurrentOfflineBrokerReplicas(broker, replicas).size();
|
||||
boolean requireLessReplicas = numOfflineTopicReplicas > 0 || numTopicReplicas > _balanceUpperLimitByTopic.get(topic) && !isExcludedForReplicaMove;
|
||||
boolean requireMoreReplicas = !isExcludedForReplicaMove && numTopicReplicas - numOfflineTopicReplicas < _balanceLowerLimitByTopic.get(topic);
|
||||
|
||||
if (requireLessReplicas) {
|
||||
rebalanceByMovingReplicasOut(broker, topic, clusterModel, optimizedGoals, optimizationOptions);
|
||||
}
|
||||
if (requireMoreReplicas) {
|
||||
rebalanceByMovingReplicasIn(broker, topic, clusterModel, optimizedGoals, optimizationOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化均衡条件:
|
||||
* 1.Topic平均分布副本数量
|
||||
* 2.Topic在平均副本的基础上向上浮动数量
|
||||
* 3.Topic在平均副本的基础上向下浮动数量
|
||||
*/
|
||||
@Override
|
||||
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
|
||||
Set<String> excludedTopics = optimizationOptions.excludedTopics();
|
||||
Set<Integer> excludedBrokers = optimizationOptions.offlineBrokers();
|
||||
Set<String> topicsAllowedRebalance = GoalUtils.getNotExcludeTopics(clusterModel, excludedTopics);
|
||||
_brokersAllowedReplicaMove = GoalUtils.getNotExcludeBrokers(clusterModel, excludedBrokers);
|
||||
if (_brokersAllowedReplicaMove.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (String topic : topicsAllowedRebalance) {
|
||||
int numTopicReplicas = clusterModel.numTopicReplicas(topic);
|
||||
_avgTopicReplicasOnBroker.put(topic, numTopicReplicas / (double) _brokersAllowedReplicaMove.size());
|
||||
_balanceUpperLimitByTopic.put(topic, balanceUpperLimit(topic, optimizationOptions));
|
||||
_balanceLowerLimitByTopic.put(topic, balanceLowerLimit(topic, optimizationOptions));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 指定Topic平均副本向下浮动,默认10%
|
||||
*/
|
||||
private Integer balanceLowerLimit(String topic, OptimizationOptions optimizationOptions) {
|
||||
return (int) Math.floor(_avgTopicReplicasOnBroker.get(topic)
|
||||
* Math.max(0, (1 - optimizationOptions.topicReplicaThreshold())));
|
||||
}
|
||||
|
||||
/**
|
||||
* 指定Topic平均副本向上浮动,默认10%
|
||||
*/
|
||||
private Integer balanceUpperLimit(String topic, OptimizationOptions optimizationOptions) {
|
||||
return (int) Math.ceil(_avgTopicReplicasOnBroker.get(topic)
|
||||
* (1 + optimizationOptions.topicReplicaThreshold()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) {
|
||||
Broker sourceBroker = clusterModel.broker(action.sourceBrokerId());
|
||||
if (sourceBroker.replica(action.topicPartition()).isCurrentOffline()) {
|
||||
return action.balancingAction() == ActionType.REPLICA_MOVEMENT;
|
||||
}
|
||||
Broker destinationBroker = clusterModel.broker(action.destinationBrokerId());
|
||||
String sourceTopic = action.topic();
|
||||
return isReplicaCountAddUpperLimit(sourceTopic, destinationBroker)
|
||||
&& (isExcludedForReplicaMove(sourceBroker) || isReplicaCountRemoveLowerLimit(sourceTopic, sourceBroker));
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return TopicReplicaDistributionGoal.class.getSimpleName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
|
||||
Broker sourceBroker = clusterModel.broker(action.sourceBrokerId());
|
||||
Broker destinationBroker = clusterModel.broker(action.destinationBrokerId());
|
||||
String sourceTopic = action.topic();
|
||||
switch (action.balancingAction()) {
|
||||
case LEADERSHIP_MOVEMENT:
|
||||
return ActionAcceptance.ACCEPT;
|
||||
case REPLICA_MOVEMENT:
|
||||
return (isReplicaCountAddUpperLimit(sourceTopic, destinationBroker)
|
||||
&& (isExcludedForReplicaMove(sourceBroker)
|
||||
|| isReplicaCountRemoveLowerLimit(sourceTopic, sourceBroker))) ? ActionAcceptance.ACCEPT : ActionAcceptance.REJECT;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported balancing action " + action.balancingAction() + " is provided.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 指定的Broker上存在Topic副本数大于阈值则迁出副本
|
||||
*/
|
||||
private boolean rebalanceByMovingReplicasOut(Broker broker,
|
||||
String topic,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
//筛选出现低于UpperLimit的所有Broker做为存放目标
|
||||
SortedSet<Broker> candidateBrokers = new TreeSet<>(
|
||||
Comparator.comparingInt((Broker b) -> b.numReplicasOfTopicInBroker(topic)).thenComparingInt(Broker::id));
|
||||
Set<Broker> filterUpperLimitBroker = clusterModel.brokers().stream().filter(b -> b.numReplicasOfTopicInBroker(topic) < _balanceUpperLimitByTopic.get(topic)).collect(Collectors.toSet());
|
||||
candidateBrokers.addAll(filterUpperLimitBroker);
|
||||
Collection<Replica> replicasOfTopicInBroker = broker.replicasOfTopicInBroker(topic);
|
||||
int numReplicasOfTopicInBroker = replicasOfTopicInBroker.size();
|
||||
int numOfflineTopicReplicas = GoalUtils.retainCurrentOfflineBrokerReplicas(broker, replicasOfTopicInBroker).size();
|
||||
int balanceUpperLimitForSourceBroker = isExcludedForReplicaMove(broker) ? 0 : _balanceUpperLimitByTopic.get(topic);
|
||||
boolean wasUnableToMoveOfflineReplica = false;
|
||||
|
||||
for (Replica replica : replicasToMoveOut(broker, topic)) {
|
||||
//当前Broker没有离线副本及Topic的副本数量低于UpperLimit则结束均衡
|
||||
if (wasUnableToMoveOfflineReplica && !replica.isCurrentOffline() && numReplicasOfTopicInBroker <= balanceUpperLimitForSourceBroker) {
|
||||
return false;
|
||||
}
|
||||
boolean wasOffline = replica.isCurrentOffline();
|
||||
Broker b = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, ActionType.REPLICA_MOVEMENT,
|
||||
optimizedGoals, optimizationOptions);
|
||||
// Only check if we successfully moved something.
|
||||
if (b != null) {
|
||||
if (wasOffline) {
|
||||
numOfflineTopicReplicas--;
|
||||
}
|
||||
if (--numReplicasOfTopicInBroker <= (numOfflineTopicReplicas == 0 ? balanceUpperLimitForSourceBroker : 0)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Remove and reinsert the broker so the order is correct.
|
||||
candidateBrokers.remove(b);
|
||||
if (b.numReplicasOfTopicInBroker(topic) < _balanceUpperLimitByTopic.get(topic)) {
|
||||
candidateBrokers.add(b);
|
||||
}
|
||||
} else if (wasOffline) {
|
||||
wasUnableToMoveOfflineReplica = true;
|
||||
}
|
||||
}
|
||||
return !broker.replicasOfTopicInBroker(topic).isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 1.离线副本优行处理
|
||||
* 2.小分区号优行处理
|
||||
*/
|
||||
private SortedSet<Replica> replicasToMoveOut(Broker broker, String topic) {
|
||||
SortedSet<Replica> replicasToMoveOut = new TreeSet<>((r1, r2) -> {
|
||||
boolean r1Offline = broker.currentOfflineReplicas().contains(r1);
|
||||
boolean r2Offline = broker.currentOfflineReplicas().contains(r2);
|
||||
if (r1Offline && !r2Offline) {
|
||||
return -1;
|
||||
} else if (!r1Offline && r2Offline) {
|
||||
return 1;
|
||||
}
|
||||
if (r1.topicPartition().partition() > r2.topicPartition().partition()) {
|
||||
return 1;
|
||||
} else if (r1.topicPartition().partition() < r2.topicPartition().partition()) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
replicasToMoveOut.addAll(broker.replicasOfTopicInBroker(topic));
|
||||
return replicasToMoveOut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Topic副本数>最低阈值的副本，迁到指定的Broker上
|
||||
*/
|
||||
private boolean rebalanceByMovingReplicasIn(Broker broker,
|
||||
String topic,
|
||||
ClusterModel clusterModel,
|
||||
Set<Goal> optimizedGoals,
|
||||
OptimizationOptions optimizationOptions) {
|
||||
PriorityQueue<Broker> eligibleBrokers = new PriorityQueue<>((b1, b2) -> {
|
||||
Collection<Replica> replicasOfTopicInB2 = b2.replicasOfTopicInBroker(topic);
|
||||
int numReplicasOfTopicInB2 = replicasOfTopicInB2.size();
|
||||
int numOfflineTopicReplicasInB2 = GoalUtils.retainCurrentOfflineBrokerReplicas(b2, replicasOfTopicInB2).size();
|
||||
Collection<Replica> replicasOfTopicInB1 = b1.replicasOfTopicInBroker(topic);
|
||||
int numReplicasOfTopicInB1 = replicasOfTopicInB1.size();
|
||||
int numOfflineTopicReplicasInB1 = GoalUtils.retainCurrentOfflineBrokerReplicas(b1, replicasOfTopicInB1).size();
|
||||
|
||||
int resultByOfflineReplicas = Integer.compare(numOfflineTopicReplicasInB2, numOfflineTopicReplicasInB1);
|
||||
if (resultByOfflineReplicas == 0) {
|
||||
int resultByAllReplicas = Integer.compare(numReplicasOfTopicInB2, numReplicasOfTopicInB1);
|
||||
return resultByAllReplicas == 0 ? Integer.compare(b1.id(), b2.id()) : resultByAllReplicas;
|
||||
}
|
||||
return resultByOfflineReplicas;
|
||||
});
|
||||
//筛选当前Topic高于LowerLimit、存在离线副本、的所有Broker做为需要迁的副本
|
||||
for (Broker sourceBroker : clusterModel.brokers()) {
|
||||
if (sourceBroker.numReplicasOfTopicInBroker(topic) > _balanceLowerLimitByTopic.get(topic)
|
||||
|| !sourceBroker.currentOfflineReplicas().isEmpty() || isExcludedForReplicaMove(sourceBroker)) {
|
||||
eligibleBrokers.add(sourceBroker);
|
||||
}
|
||||
}
|
||||
Collection<Replica> replicasOfTopicInBroker = broker.replicasOfTopicInBroker(topic);
|
||||
int numReplicasOfTopicInBroker = replicasOfTopicInBroker.size();
|
||||
//当前Broker做为存放目标
|
||||
Set<Broker> candidateBrokers = Collections.singleton(broker);
|
||||
while (!eligibleBrokers.isEmpty()) {
|
||||
Broker sourceBroker = eligibleBrokers.poll();
|
||||
SortedSet<Replica> replicasToMove = replicasToMoveOut(sourceBroker, topic);
|
||||
int numOfflineTopicReplicas = GoalUtils.retainCurrentOfflineBrokerReplicas(sourceBroker, replicasToMove).size();
|
||||
|
||||
for (Replica replica : replicasToMove) {
|
||||
boolean wasOffline = replica.isCurrentOffline();
|
||||
Broker b = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, ActionType.REPLICA_MOVEMENT,
|
||||
optimizedGoals, optimizationOptions);
|
||||
if (b != null) {
|
||||
if (wasOffline) {
|
||||
numOfflineTopicReplicas--;
|
||||
}
|
||||
if (++numReplicasOfTopicInBroker >= _balanceLowerLimitByTopic.get(topic)) {
|
||||
return false;
|
||||
}
|
||||
if (!eligibleBrokers.isEmpty() && numOfflineTopicReplicas == 0
|
||||
&& sourceBroker.numReplicasOfTopicInBroker(topic) < eligibleBrokers.peek().numReplicasOfTopicInBroker(topic)) {
|
||||
eligibleBrokers.add(sourceBroker);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 目标Broker增加副本后，Topic副本数<=最高阈值
|
||||
*/
|
||||
private boolean isReplicaCountAddUpperLimit(String topic, Broker destinationBroker) {
|
||||
int numTopicReplicas = destinationBroker.numReplicasOfTopicInBroker(topic);
|
||||
int brokerBalanceUpperLimit = _balanceUpperLimitByTopic.get(topic);
|
||||
return numTopicReplicas + 1 <= brokerBalanceUpperLimit;
|
||||
}
|
||||
|
||||
/**
|
||||
* 源Broker迁走副本后，Topic副本数>=最低阈值
|
||||
*/
|
||||
private boolean isReplicaCountRemoveLowerLimit(String topic, Broker sourceBroker) {
|
||||
int numTopicReplicas = sourceBroker.numReplicasOfTopicInBroker(topic);
|
||||
int brokerBalanceLowerLimit = _balanceLowerLimitByTopic.get(topic);
|
||||
return numTopicReplicas - 1 >= brokerBalanceLowerLimit;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断指定的Broker是否可以进行副本迁移操作
|
||||
*/
|
||||
private boolean isExcludedForReplicaMove(Broker broker) {
|
||||
return !_brokersAllowedReplicaMove.contains(broker);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断指定的Topic是否在可均衡的列表中
|
||||
*/
|
||||
private boolean isTopicExcludedFromRebalance(String topic) {
|
||||
return _avgTopicReplicasOnBroker.get(topic) == null;
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.utils;
|
||||
|
||||
import joptsimple.OptionParser;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class CommandLineUtils {
|
||||
|
||||
/**
|
||||
* Print usage and exit
|
||||
*/
|
||||
public static void printUsageAndDie(OptionParser parser, String message) {
|
||||
try {
|
||||
System.err.println(message);
|
||||
parser.printHelpOn(System.err);
|
||||
System.exit(1);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.utils;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceGoal;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceThreshold;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.HostEnv;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.*;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.AnalyzerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class GoalUtils {
|
||||
private static final Logger logger = LoggerFactory.getLogger(GoalUtils.class);
|
||||
|
||||
public static Set<String> getNotExcludeTopics(ClusterModel clusterModel, Set<String> excludedTopics) {
|
||||
return clusterModel.topics().stream().filter(topicName -> !excludedTopics.contains(topicName)).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public static Set<Broker> getNotExcludeBrokers(ClusterModel clusterModel, Set<Integer> excludedBrokers) {
|
||||
return clusterModel.brokers().stream().filter(broker -> !excludedBrokers.contains(broker.id())).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
/**
|
||||
* 在Broker上获取指定的离线副本列表
|
||||
*/
|
||||
public static Set<Replica> retainCurrentOfflineBrokerReplicas(Broker broker, Collection<Replica> replicas) {
|
||||
Set<Replica> offlineReplicas = new HashSet<>(replicas);
|
||||
offlineReplicas.retainAll(broker.currentOfflineReplicas());
|
||||
return offlineReplicas;
|
||||
}
|
||||
|
||||
public static ClusterModel getInitClusterModel(BalanceParameter parameter) {
|
||||
logger.info("Cluster model initialization");
|
||||
List<HostEnv> hostsEnv = parameter.getHardwareEnv();
|
||||
Map<Integer, Capacity> capacities = new HashMap<>();
|
||||
for (HostEnv env : hostsEnv) {
|
||||
Capacity capacity = new Capacity();
|
||||
capacity.setCapacity(Resource.CPU, env.getCpu());
|
||||
capacity.setCapacity(Resource.DISK, env.getDisk());
|
||||
capacity.setCapacity(Resource.NW_IN, env.getNetwork());
|
||||
capacity.setCapacity(Resource.NW_OUT, env.getNetwork());
|
||||
capacities.put(env.getId(), capacity);
|
||||
}
|
||||
return Supplier.load(parameter.getCluster(), parameter.getBeforeSeconds(), parameter.getKafkaConfig(),
|
||||
parameter.getEsRestURL(), parameter.getEsIndexPrefix(), capacities, AnalyzerUtils.getSplitTopics(parameter.getIgnoredTopics()));
|
||||
}
|
||||
|
||||
public static Map<String, BalanceThreshold> getBalanceThreshold(BalanceParameter parameter, double[] clusterAvgResource) {
|
||||
Map<String, BalanceThreshold> balanceThreshold = new HashMap<>();
|
||||
balanceThreshold.put(BalanceGoal.DISK.goal(), new BalanceThreshold(Resource.DISK, parameter.getDiskThreshold(), clusterAvgResource[Resource.DISK.id()]));
|
||||
balanceThreshold.put(BalanceGoal.NW_IN.goal(), new BalanceThreshold(Resource.NW_IN, parameter.getNetworkInThreshold(), clusterAvgResource[Resource.NW_IN.id()]));
|
||||
balanceThreshold.put(BalanceGoal.NW_OUT.goal(), new BalanceThreshold(Resource.NW_OUT, parameter.getNetworkOutThreshold(), clusterAvgResource[Resource.NW_OUT.id()]));
|
||||
return balanceThreshold;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance.utils;
|
||||
|
||||
import org.apache.kafka.clients.*;
|
||||
import org.apache.kafka.clients.consumer.internals.NoAvailableBrokersException;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.common.Cluster;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.network.ChannelBuilder;
|
||||
import org.apache.kafka.common.network.NetworkReceive;
|
||||
import org.apache.kafka.common.network.Selector;
|
||||
import org.apache.kafka.common.requests.MetadataRequest;
|
||||
import org.apache.kafka.common.requests.MetadataResponse;
|
||||
import org.apache.kafka.common.utils.LogContext;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.apache.kafka.common.utils.Utils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/27
|
||||
*/
|
||||
public class MetadataUtils {
|
||||
private static final Logger logger = LoggerFactory.getLogger(MetadataUtils.class);
|
||||
|
||||
public static Cluster metadata(Properties props) {
|
||||
props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.BytesSerializer");
|
||||
props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.BytesSerializer");
|
||||
ProducerConfig config = new ProducerConfig(props);
|
||||
|
||||
Time time = Time.SYSTEM;
|
||||
LogContext logContext = new LogContext("Metadata client");
|
||||
|
||||
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time, logContext);
|
||||
Selector selector = new Selector(
|
||||
NetworkReceive.UNLIMITED,
|
||||
config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG),
|
||||
new org.apache.kafka.common.metrics.Metrics(),
|
||||
time,
|
||||
"metadata-client",
|
||||
Collections.singletonMap("client", "metadata-client"),
|
||||
false,
|
||||
channelBuilder,
|
||||
logContext
|
||||
);
|
||||
|
||||
NetworkClient networkClient = new NetworkClient(
|
||||
selector,
|
||||
new ManualMetadataUpdater(),
|
||||
"metadata-client",
|
||||
1,
|
||||
config.getLong(ProducerConfig.RECONNECT_BACKOFF_MS_CONFIG),
|
||||
config.getLong(ProducerConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG),
|
||||
config.getInt(ProducerConfig.SEND_BUFFER_CONFIG),
|
||||
config.getInt(ProducerConfig.RECEIVE_BUFFER_CONFIG),
|
||||
config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG),
|
||||
config.getLong(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG),
|
||||
config.getLong(ProducerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG),
|
||||
ClientDnsLookup.DEFAULT,
|
||||
time,
|
||||
true,
|
||||
new ApiVersions(),
|
||||
logContext
|
||||
);
|
||||
|
||||
try {
|
||||
List<String> nodes = config.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
|
||||
for (int i = 0; i < nodes.size(); i++) {
|
||||
Node sourceNode = new Node(i, Utils.getHost(nodes.get(i)), Utils.getPort(nodes.get(i)));
|
||||
try {
|
||||
if (NetworkClientUtils.awaitReady(networkClient, sourceNode, time, 10 * 1000)) {
|
||||
ClientRequest clientRequest = networkClient.newClientRequest(String.valueOf(i), MetadataRequest.Builder.allTopics(),
|
||||
time.milliseconds(), true);
|
||||
ClientResponse clientResponse = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, time);
|
||||
MetadataResponse metadataResponse = (MetadataResponse) clientResponse.responseBody();
|
||||
return metadataResponse.buildCluster();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warn("Connection to " + sourceNode + " error", e);
|
||||
}
|
||||
}
|
||||
throw new NoAvailableBrokersException();
|
||||
} finally {
|
||||
networkClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.TopicLeadersDistributionGoal
|
||||
com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.TopicReplicaDistributionGoal
|
||||
com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.DiskDistributionGoal
|
||||
com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.NetworkInboundDistributionGoal
|
||||
com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.NetworkOutboundDistributionGoal
|
||||
@@ -1,75 +0,0 @@
|
||||
{
|
||||
"size": 0,
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"clusterPhyId": "<var_cluster_name>"
|
||||
}
|
||||
},
|
||||
{
|
||||
"range": {
|
||||
"timestamp": {
|
||||
"gte": "now-<var_before_time>s",
|
||||
"lte": "now"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"aggs": {
|
||||
"by_topic": {
|
||||
"terms": {
|
||||
"field": "topic",
|
||||
"size": 5000
|
||||
},
|
||||
"aggs": {
|
||||
"by_partition": {
|
||||
"terms": {
|
||||
"field": "partitionId",
|
||||
"size": 2000
|
||||
},
|
||||
"aggs": {
|
||||
"avg_cpu": {
|
||||
"avg": {
|
||||
"field": "metrics.CPU"
|
||||
}
|
||||
},
|
||||
"avg_bytes_in": {
|
||||
"avg": {
|
||||
"field": "metrics.BytesIn"
|
||||
}
|
||||
},
|
||||
"avg_bytes_out": {
|
||||
"avg": {
|
||||
"field": "metrics.BytesOut"
|
||||
}
|
||||
},
|
||||
"avg_disk": {
|
||||
"avg": {
|
||||
"field": "metrics.LogSize"
|
||||
}
|
||||
},
|
||||
"lastest_disk": {
|
||||
"top_hits": {
|
||||
"sort": [
|
||||
{
|
||||
"timestamp": {
|
||||
"order": "desc"
|
||||
}
|
||||
}
|
||||
],
|
||||
"_source": {
|
||||
"includes": [ "metrics.LogSize"]
|
||||
},
|
||||
"size": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package com.xiaojukeji.know.streaming.km.rebalance;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.ExecutionRebalance;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.BalanceParameter;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.HostEnv;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.ClusterModel;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.model.Replica;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.executor.common.OptimizerResult;
|
||||
import com.xiaojukeji.know.streaming.km.rebalance.optimizer.goals.Goal;
|
||||
import org.apache.kafka.clients.CommonClientConfigs;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author leewei
|
||||
* @date 2022/5/30
|
||||
*/
|
||||
public class Test {
|
||||
private static HostEnv getHostEnv(int id) {
|
||||
HostEnv env = new HostEnv();
|
||||
env.setId(id);
|
||||
env.setCpu(3200); // 32C
|
||||
env.setDisk(2 * 1024D * 1024 * 1024 * 1024); // 2T
|
||||
env.setNetwork(10 * 1024D * 1024 * 1024); // 10G
|
||||
return env;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
ServiceLoader<Goal> loader = ServiceLoader.load(Goal.class);
|
||||
for (Goal goal : loader) {
|
||||
System.out.println(goal.name() + " ： " + goal);
|
||||
}
|
||||
|
||||
BalanceParameter balanceParameter = new BalanceParameter();
|
||||
|
||||
balanceParameter.setDiskThreshold(0.05);
|
||||
balanceParameter.setCpuThreshold(0.05);
|
||||
balanceParameter.setNetworkInThreshold(0.05);
|
||||
balanceParameter.setNetworkOutThreshold(0.05);
|
||||
|
||||
List<HostEnv> envList = new ArrayList<>();
|
||||
envList.add(getHostEnv(6416));
|
||||
envList.add(getHostEnv(6417));
|
||||
envList.add(getHostEnv(6431));
|
||||
envList.add(getHostEnv(6432));
|
||||
envList.add(getHostEnv(6553));
|
||||
balanceParameter.setHardwareEnv(envList);
|
||||
|
||||
// String goals = "DiskDistributionGoal,NetworkInboundDistributionGoal,NetworkOutboundDistributionGoal";
|
||||
// String goals = "DiskDistributionGoal";
|
||||
String goals = "NetworkOutboundDistributionGoal";
|
||||
|
||||
balanceParameter.setGoals(Arrays.asList(goals.split(",")));
|
||||
Properties kafkaConfig = new Properties();
|
||||
kafkaConfig.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "10.96.64.16:7262");
|
||||
// kafkaConfig.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "10.96.64.16:9162,10.96.64.17:9162,10.96.64.31:9162");
|
||||
balanceParameter.setKafkaConfig(kafkaConfig);
|
||||
balanceParameter.setEsRestURL("10.96.64.13:8061");
|
||||
balanceParameter.setEsIndexPrefix("ks_kafka_partition_metric_");
|
||||
|
||||
balanceParameter.setBeforeSeconds(300);
|
||||
|
||||
balanceParameter.setCluster("293");
|
||||
|
||||
ExecutionRebalance exec = new ExecutionRebalance();
|
||||
OptimizerResult optimizerResult = exec.optimizations(balanceParameter);
|
||||
|
||||
System.out.println(optimizerResult.resultJsonOverview());
|
||||
System.out.println(optimizerResult.resultJsonDetailed());
|
||||
System.out.println(optimizerResult.resultJsonTask());
|
||||
System.out.println(optimizerResult.resultJsonBalanceActionHistory());
|
||||
|
||||
ClusterModel model = optimizerResult.clusterModel();
|
||||
|
||||
System.out.println("---moved partitions----");
|
||||
// moved partitions
|
||||
for (String topic : model.topics()) {
|
||||
model.topic(topic).stream()
|
||||
.filter(Partition::isChanged)
|
||||
.forEach(partition -> {
|
||||
System.out.println("---> " + partition);
|
||||
System.out.println(partition.topicPartition() + " leader change: " + partition.isLeaderChanged() + " " + (partition.isLeaderChanged() ? partition.originalLeaderBroker().id() + " -> " + partition.leader().broker().id() : ""));
|
||||
partition.replicas().stream()
|
||||
.filter(Replica::isChanged).forEach(r -> {
|
||||
//System.out.println(r);
|
||||
System.out.println(partition.topicPartition() + " replica moved: " + r.original().broker().id() + "（" + r.original().isLeader() + "） -> " + r.broker().id() + " (" + r.isLeader() + ")");
|
||||
});
|
||||
});
|
||||
}
|
||||
System.out.println("---end moved partitions----");
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
log4j.rootLogger=INFO, stdout
|
||||
|
||||
# Console output...
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n
|
||||
Reference in New Issue
Block a user