8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2026-01-07 14:34:52 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

开放接口&近期BUG修复

Browse Source
This commit is contained in:
zengqiao
2020-10-26 11:17:45 +08:00
parent 8b153113ff
commit a77242e66c
62 changed files with 12138 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/common/entry/N9eResult.java Normal file
View File

@@ -0,0 +1,35 @@
package com.xiaojukeji.kafka.manager.account.common.entry;
/**
* @author zengqiao
* @date 20/9/7
*/
public class N9eResult<T> {
private T dat;
private String err;
public T getDat() {
return dat;
}
public void setDat(T dat) {
this.dat = dat;
}
public String getErr() {
return err;
}
public void setErr(String err) {
this.err = err;
}
@Override
public String toString() {
return "N9eResult{" +
"dat=" + dat +
", err='" + err + '\'' +
'}';
}
}

134
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/common/entry/N9eUserData.java Normal file
View File

@@ -0,0 +1,134 @@
package com.xiaojukeji.kafka.manager.account.common.entry;
/**
* @author zengqiao
* @date 20/9/7
*/
public class N9eUserData {
private Long id;
private String uuid;
private String username;
private String dispname;
private String phone;
private String email;
private String im;
private String portrait;
private Integer is_root;
private Integer leader_id;
private String leader_name;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getUuid() {
return uuid;
}
public void setUuid(String uuid) {
this.uuid = uuid;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getDispname() {
return dispname;
}
public void setDispname(String dispname) {
this.dispname = dispname;
}
public String getPhone() {
return phone;
}
public void setPhone(String phone) {
this.phone = phone;
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
}
public String getIm() {
return im;
}
public void setIm(String im) {
this.im = im;
}
public String getPortrait() {
return portrait;
}
public void setPortrait(String portrait) {
this.portrait = portrait;
}
public Integer getIs_root() {
return is_root;
}
public void setIs_root(Integer is_root) {
this.is_root = is_root;
}
public Integer getLeader_id() {
return leader_id;
}
public void setLeader_id(Integer leader_id) {
this.leader_id = leader_id;
}
public String getLeader_name() {
return leader_name;
}
public void setLeader_name(String leader_name) {
this.leader_name = leader_name;
}
@Override
public String toString() {
return "EPRIResult{" +
"id=" + id +
", uuid='" + uuid + '\'' +
", username='" + username + '\'' +
", dispname='" + dispname + '\'' +
", phone='" + phone + '\'' +
", email='" + email + '\'' +
", im='" + im + '\'' +
", portrait='" + portrait + '\'' +
", is_root=" + is_root +
", leader_id=" + leader_id +
", leader_name='" + leader_name + '\'' +
'}';
}
}

2
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/account/BaseEnterpriseStaffService.java
View File

@@ -55,4 +55,4 @@ public class BaseEnterpriseStaffService extends AbstractEnterpriseStaffService {
}
return new ArrayList<>();
}
}
}

10
kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/N9e.java
View File

@@ -30,19 +30,19 @@ import java.util.Map;
public class N9e extends AbstractAgent {
private static final Logger LOGGER = LoggerFactory.getLogger(N9e.class);
@Value("${agent.n9e.base-url}")
@Value("${kcm.n9e.base-url}")
private String baseUrl;
@Value("${agent.n9e.username}")
@Value("${kcm.n9e.username}")
private String username;
@Value("${agent.n9e.user-token}")
@Value("${kcm.n9e.user-token}")
private String userToken;
@Value("${agent.n9e.tpl-id}")
@Value("${kcm.n9e.tpl-id}")
private Integer tplId;
@Value("${agent.n9e.timeout}")
@Value("${kcm.n9e.timeout}")
private Integer timeout;
/**

7
kafka-manager-extends/kafka-manager-kcm/src/main/resources/application-kcm-dev.yml
View File

@@ -1,7 +0,0 @@
agent:
n9e:
base-url: http://127.0.0.1/api
username: admin
user-token: admin
tpl-id: 123456
timeout: 30

129
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/N9eConverter.java
View File

@@ -1,9 +1,11 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e;
import com.xiaojukeji.kafka.manager.monitor.common.entry.MetricSinkPoint;
import com.xiaojukeji.kafka.manager.monitor.component.n9e.entry.N9eMetricSinkPoint;
import com.xiaojukeji.kafka.manager.common.utils.ListUtils;
import com.xiaojukeji.kafka.manager.monitor.common.entry.*;
import com.xiaojukeji.kafka.manager.monitor.component.n9e.entry.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
@@ -27,4 +29,127 @@ public class N9eConverter {
}
return n9ePointList;
}
public static N9eStrategy convert2N9eStrategy(Strategy strategy, Integer monitorN9eNid) {
if (strategy == null) {
return null;
}
N9eStrategy n9eStrategy = new N9eStrategy();
n9eStrategy.setId(strategy.getId().intValue());
n9eStrategy.setCategory(1);
n9eStrategy.setName(strategy.getName());
n9eStrategy.setNid(monitorN9eNid);
n9eStrategy.setExcl_nid(new ArrayList<>());
n9eStrategy.setPriority(strategy.getPriority());
n9eStrategy.setAlert_dur(60);
List<N9eStrategyExpression> exprs = new ArrayList<>();
for (StrategyExpression strategyExpression: strategy.getStrategyExpressionList()) {
N9eStrategyExpression n9eStrategyExpression = new N9eStrategyExpression();
n9eStrategyExpression.setMetric(strategyExpression.getMetric());
n9eStrategyExpression.setFunc(strategyExpression.getFunc());
n9eStrategyExpression.setEopt(strategyExpression.getEopt());
n9eStrategyExpression.setThreshold(strategyExpression.getThreshold().intValue());
n9eStrategyExpression.setParams(ListUtils.string2IntList(strategyExpression.getParams()));
exprs.add(n9eStrategyExpression);
}
n9eStrategy.setExprs(exprs);
List<N9eStrategyFilter> tags = new ArrayList<>();
for (StrategyFilter strategyFilter: strategy.getStrategyFilterList()) {
N9eStrategyFilter n9eStrategyFilter = new N9eStrategyFilter();
n9eStrategyFilter.setTkey(strategyFilter.getTkey());
n9eStrategyFilter.setTopt(strategyFilter.getTopt());
n9eStrategyFilter.setTval(Arrays.asList(strategyFilter.getTval()));
tags.add(n9eStrategyFilter);
}
n9eStrategy.setTags(tags);
n9eStrategy.setRecovery_dur(0);
n9eStrategy.setRecovery_notify(0);
StrategyAction strategyAction = strategy.getStrategyActionList().get(0);
n9eStrategy.setConverge(ListUtils.string2IntList(strategyAction.getConverge()));
n9eStrategy.setNotify_group(ListUtils.string2StrList(strategyAction.getNotifyGroup()));
n9eStrategy.setNotify_user(new ArrayList<>());
n9eStrategy.setCallback(strategyAction.getCallback());
n9eStrategy.setEnable_stime("00:00");
n9eStrategy.setEnable_etime("23:59");
n9eStrategy.setEnable_days_of_week(ListUtils.string2IntList(strategy.getPeriodDaysOfWeek()));
n9eStrategy.setNeed_upgrade(0);
n9eStrategy.setAlert_upgrade(new ArrayList<>());
return n9eStrategy;
}
public static List<Strategy> convert2StrategyList(List<N9eStrategy> n9eStrategyList) {
if (n9eStrategyList == null || n9eStrategyList.isEmpty()) {
return new ArrayList<>();
}
List<Strategy> strategyList = new ArrayList<>();
for (N9eStrategy n9eStrategy: n9eStrategyList) {
strategyList.add(convert2Strategy(n9eStrategy));
}
return strategyList;
}
public static Strategy convert2Strategy(N9eStrategy n9eStrategy) {
if (n9eStrategy == null) {
return null;
}
Strategy strategy = new Strategy();
strategy.setId(n9eStrategy.getId().longValue());
strategy.setName(n9eStrategy.getName());
strategy.setPriority(n9eStrategy.getPriority());
strategy.setPeriodHoursOfDay("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23");
strategy.setPeriodDaysOfWeek(ListUtils.intList2String(n9eStrategy.getEnable_days_of_week()));
List<StrategyExpression> strategyExpressionList = new ArrayList<>();
for (N9eStrategyExpression n9eStrategyExpression: n9eStrategy.getExprs()) {
StrategyExpression strategyExpression = new StrategyExpression();
strategyExpression.setMetric(n9eStrategyExpression.getMetric());
strategyExpression.setFunc(n9eStrategyExpression.getFunc());
strategyExpression.setEopt(n9eStrategyExpression.getEopt());
strategyExpression.setThreshold(n9eStrategyExpression.getThreshold().longValue());
strategyExpression.setParams(ListUtils.intList2String(n9eStrategyExpression.getParams()));
strategyExpressionList.add(strategyExpression);
}
strategy.setStrategyExpressionList(strategyExpressionList);
List<StrategyFilter> strategyFilterList = new ArrayList<>();
for (N9eStrategyFilter n9eStrategyFilter: n9eStrategy.getTags()) {
StrategyFilter strategyFilter = new StrategyFilter();
strategyFilter.setTkey(n9eStrategyFilter.getTkey());
strategyFilter.setTopt(n9eStrategyFilter.getTopt());
strategyFilter.setTval(ListUtils.strList2String(n9eStrategyFilter.getTval()));
strategyFilterList.add(strategyFilter);
}
strategy.setStrategyFilterList(strategyFilterList);
StrategyAction strategyAction = new StrategyAction();
strategyAction.setNotifyGroup(ListUtils.strList2String(n9eStrategy.getNotify_group()));
strategyAction.setConverge(ListUtils.intList2String(n9eStrategy.getConverge()));
strategyAction.setCallback(n9eStrategy.getCallback());
strategy.setStrategyActionList(Arrays.asList(strategyAction));
return strategy;
}
public static List<NotifyGroup> convert2NotifyGroupList(N9eNotifyGroup n9eNotifyGroup) {
if (n9eNotifyGroup == null || n9eNotifyGroup.getList() == null) {
return new ArrayList<>();
}
List<NotifyGroup> notifyGroupList = new ArrayList<>();
for (N9eNotifyGroupElem n9eNotifyGroupElem: n9eNotifyGroup.getList()) {
NotifyGroup notifyGroup = new NotifyGroup();
notifyGroup.setId(n9eNotifyGroupElem.getId().longValue());
notifyGroup.setName(n9eNotifyGroupElem.getName());
notifyGroup.setComment(n9eNotifyGroupElem.getNote());
notifyGroupList.add(notifyGroup);
}
return notifyGroupList;
}
}

138
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/N9eService.java
View File

@@ -2,17 +2,18 @@ package com.xiaojukeji.kafka.manager.monitor.component.n9e;
import com.alibaba.fastjson.JSON;
import com.xiaojukeji.kafka.manager.common.utils.HttpUtils;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.monitor.component.AbstractMonitorService;
import com.xiaojukeji.kafka.manager.monitor.common.entry.*;
import com.xiaojukeji.kafka.manager.monitor.component.n9e.entry.N9eNotifyGroup;
import com.xiaojukeji.kafka.manager.monitor.component.n9e.entry.N9eResult;
import com.xiaojukeji.kafka.manager.monitor.component.n9e.entry.N9eStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.*;
/**
* 夜莺
@@ -23,21 +24,28 @@ import java.util.Properties;
public class N9eService extends AbstractMonitorService {
private static final Logger LOGGER = LoggerFactory.getLogger(N9eService.class);
@Value("${monitor.n9e.nid}")
private Integer monitorN9eNid;
@Value("${monitor.n9e.user-token}")
private String monitorN9eToken;
@Value("${monitor.n9e.base-url}")
private String monitorN9eBaseUrl;
/**
* 告警策略
*/
private static final String STRATEGY_ADD_URL = "/auth/v1/strategy/add";
private static final String STRATEGY_ADD_URL = "/api/mon/stra";
private static final String STRATEGY_DEL_URL = "/auth/v1/strategy/del";
private static final String STRATEGY_DEL_URL = "/api/mon/stra";
private static final String STRATEGY_MODIFY_URL = "/auth/v1/strategy/modify";
private static final String STRATEGY_MODIFY_URL = "/api/mon/stra";
private static final String STRATEGY_QUERY_BY_NS_URL = "/auth/v1/strategy/query/ns";
private static final String STRATEGY_QUERY_BY_NS_URL = "/api/mon/stra";
private static final String STRATEGY_QUERY_BY_ID_URL = "/api/mon/stra";
private static final String STRATEGY_QUERY_BY_ID_URL = "/auth/v1/strategy/query/id";
private static final String ALERT_QUERY_BY_NS_AND_PERIOD_URL = "/auth/v1/event/query/ns/period";
@@ -57,41 +65,121 @@ public class N9eService extends AbstractMonitorService {
/**
* 指标数据
*/
private static final String COLLECTOR_SINK_DATA_URL = "/api/collector/push";
private static final String COLLECTOR_SINK_DATA_URL = "/api/transfer/push";
private static final String COLLECTOR_DOWNLOAD_DATA_URL = "/data/query/graph/dashboard/history";
/**
* 告警组
*/
private static final String ALL_NOTIFY_GROUP_URL = "/auth/v1/usergroup/group/all";
private static final String ALL_NOTIFY_GROUP_URL = "/api/mon/teams/all";
/**
* 监控策略的增删改查
*/
@Override
public Integer createStrategy(Strategy strategy) {
return 0;
String response = null;
try {
response = HttpUtils.postForString(
monitorN9eBaseUrl + STRATEGY_ADD_URL,
JSON.toJSONString(N9eConverter.convert2N9eStrategy(strategy, monitorN9eNid)),
buildHeader()
);
N9eResult n9eResult = JSON.parseObject(response, N9eResult.class);
if (!ValidateUtils.isBlank(n9eResult.getErr())) {
LOGGER.error("create strategy failed, strategy:{} response:{}.", strategy, response);
return null;
}
return (Integer) n9eResult.getDat();
} catch (Exception e) {
LOGGER.error("create strategy failed, strategy:{} response:{}.", strategy, response, e);
}
return null;
}
@Override
public Boolean deleteStrategyById(Long strategyId) {
return true;
Map<String, List<Long>> params = new HashMap<>(1);
params.put("ids", Arrays.asList(strategyId));
String response = null;
try {
response = HttpUtils.deleteForString(
monitorN9eBaseUrl + STRATEGY_DEL_URL,
JSON.toJSONString(params),
buildHeader()
);
N9eResult n9eResult = JSON.parseObject(response, N9eResult.class);
if (!ValidateUtils.isBlank(n9eResult.getErr())) {
LOGGER.error("delete strategy failed, strategyId:{} response:{}.", strategyId, response);
return Boolean.FALSE;
}
return Boolean.TRUE;
} catch (Exception e) {
LOGGER.error("delete strategy failed, strategyId:{} response:{}.", strategyId, response, e);
}
return Boolean.FALSE;
}
@Override
public Boolean modifyStrategy(Strategy strategy) {
return true;
String response = null;
try {
response = HttpUtils.putForString(
monitorN9eBaseUrl + STRATEGY_MODIFY_URL,
JSON.toJSONString(N9eConverter.convert2N9eStrategy(strategy, monitorN9eNid)),
buildHeader()
);
N9eResult n9eResult = JSON.parseObject(response, N9eResult.class);
if (!ValidateUtils.isBlank(n9eResult.getErr())) {
LOGGER.error("modify strategy failed, strategy:{} response:{}.", strategy, response);
return Boolean.FALSE;
}
return Boolean.TRUE;
} catch (Exception e) {
LOGGER.error("modify strategy failed, strategy:{} response:{}.", strategy, response, e);
}
return Boolean.FALSE;
}
@Override
public List<Strategy> getStrategies() {
Map<String, String> params = new HashMap<>();
params.put("nid", String.valueOf(monitorN9eNid));
String response = null;
try {
response = HttpUtils.get(monitorN9eBaseUrl + STRATEGY_QUERY_BY_NS_URL, params, buildHeader());
N9eResult n9eResult = JSON.parseObject(response, N9eResult.class);
if (!ValidateUtils.isBlank(n9eResult.getErr())) {
LOGGER.error("get monitor strategies failed, response:{}.", response);
return new ArrayList<>();
}
return N9eConverter.convert2StrategyList(JSON.parseArray(JSON.toJSONString(n9eResult.getDat()), N9eStrategy.class));
} catch (Exception e) {
LOGGER.error("get monitor strategies failed, response:{}.", response, e);
}
return new ArrayList<>();
}
@Override
public Strategy getStrategyById(Long strategyId) {
return new Strategy();
String uri = STRATEGY_QUERY_BY_ID_URL + "/" + String.valueOf(strategyId);
String response = null;
try {
response = HttpUtils.get(monitorN9eBaseUrl + uri, new HashMap<>(0), buildHeader());
N9eResult n9eResult = JSON.parseObject(response, N9eResult.class);
if (!ValidateUtils.isBlank(n9eResult.getErr())) {
LOGGER.error("get monitor strategy failed, response:{}.", response);
return null;
}
return N9eConverter.convert2Strategy(JSON.parseObject(JSON.toJSONString(n9eResult.getDat()), N9eStrategy.class));
} catch (Exception e) {
LOGGER.error("get monitor strategy failed, response:{}.", response, e);
}
return null;
}
@Override
@@ -161,6 +249,26 @@ public class N9eService extends AbstractMonitorService {
@Override
public List<NotifyGroup> getNotifyGroups() {
String response = null;
try {
response = HttpUtils.get(monitorN9eBaseUrl + ALL_NOTIFY_GROUP_URL, new HashMap<>(0), buildHeader());
N9eResult n9eResult = JSON.parseObject(response, N9eResult.class);
if (!ValidateUtils.isBlank(n9eResult.getErr())) {
LOGGER.error("get notify group failed, response:{}.", response);
return new ArrayList<>();
}
return N9eConverter.convert2NotifyGroupList(JSON.parseObject(JSON.toJSONString(n9eResult.getDat()), N9eNotifyGroup.class));
} catch (Exception e) {
LOGGER.error("get notify group failed, response:{}.", response, e);
}
return new ArrayList<>();
}
}
private Map<String, String> buildHeader() {
Map<String, String> header = new HashMap<>(2);
header.put("Content-Type", "application/json");
header.put("X-User-Token", monitorN9eToken);
return header;
}
}

26
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/entry/N9eNotifyGroup.java Normal file
View File

@@ -0,0 +1,26 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e.entry;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/19
*/
public class N9eNotifyGroup {
private List<N9eNotifyGroupElem> list;
public List<N9eNotifyGroupElem> getList() {
return list;
}
public void setList(List<N9eNotifyGroupElem> list) {
this.list = list;
}
@Override
public String toString() {
return "N9eNotifyGroup{" +
"list=" + list +
'}';
}
}

90
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/entry/N9eNotifyGroupElem.java Normal file
View File

@@ -0,0 +1,90 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e.entry;
/**
* @author zengqiao
* @date 20/10/19
*/
public class N9eNotifyGroupElem {
private Integer creator;
private Integer id;
private String ident;
private String last_updated;
private Integer mgmt;
private String name;
private String note;
public Integer getCreator() {
return creator;
}
public void setCreator(Integer creator) {
this.creator = creator;
}
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getIdent() {
return ident;
}
public void setIdent(String ident) {
this.ident = ident;
}
public String getLast_updated() {
return last_updated;
}
public void setLast_updated(String last_updated) {
this.last_updated = last_updated;
}
public Integer getMgmt() {
return mgmt;
}
public void setMgmt(Integer mgmt) {
this.mgmt = mgmt;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getNote() {
return note;
}
public void setNote(String note) {
this.note = note;
}
@Override
public String toString() {
return "N9eNotifyGroupElem{" +
"creator=" + creator +
", id=" + id +
", ident='" + ident + '\'' +
", last_updated='" + last_updated + '\'' +
", mgmt=" + mgmt +
", name='" + name + '\'' +
", note='" + note + '\'' +
'}';
}
}

242
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/entry/N9eStrategy.java Normal file
View File

@@ -0,0 +1,242 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e.entry;
import java.util.ArrayList;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/18
*/
public class N9eStrategy {
private Integer id;
private Integer category = 1;
/**
* 策略名称
*/
private String name;
/**
* 策略关联的对象树节点id
*/
private Integer nid;
private List<Integer> excl_nid = new ArrayList<>();
private Integer priority;
private Integer alert_dur = 60;
private List<N9eStrategyExpression> exprs;
private List<N9eStrategyFilter> tags;
private Integer recovery_dur;
private Integer recovery_notify;
private List<N9eStrategyAlertUpgrade> alert_upgrade = new ArrayList<>();
private List<Integer> converge;
private List<String> notify_group;
private List<Integer> notify_user;
private String callback;
private String enable_stime;
private String enable_etime;
private List<Integer> enable_days_of_week;
private Integer need_upgrade;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public Integer getCategory() {
return category;
}
public void setCategory(Integer category) {
this.category = category;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getNid() {
return nid;
}
public void setNid(Integer nid) {
this.nid = nid;
}
public List<Integer> getExcl_nid() {
return excl_nid;
}
public void setExcl_nid(List<Integer> excl_nid) {
this.excl_nid = excl_nid;
}
public Integer getPriority() {
return priority;
}
public void setPriority(Integer priority) {
this.priority = priority;
}
public Integer getAlert_dur() {
return alert_dur;
}
public void setAlert_dur(Integer alert_dur) {
this.alert_dur = alert_dur;
}
public List<N9eStrategyExpression> getExprs() {
return exprs;
}
public void setExprs(List<N9eStrategyExpression> exprs) {
this.exprs = exprs;
}
public List<N9eStrategyFilter> getTags() {
return tags;
}
public void setTags(List<N9eStrategyFilter> tags) {
this.tags = tags;
}
public Integer getRecovery_dur() {
return recovery_dur;
}
public void setRecovery_dur(Integer recovery_dur) {
this.recovery_dur = recovery_dur;
}
public Integer getRecovery_notify() {
return recovery_notify;
}
public void setRecovery_notify(Integer recovery_notify) {
this.recovery_notify = recovery_notify;
}
public List<N9eStrategyAlertUpgrade> getAlert_upgrade() {
return alert_upgrade;
}
public void setAlert_upgrade(List<N9eStrategyAlertUpgrade> alert_upgrade) {
this.alert_upgrade = alert_upgrade;
}
public List<Integer> getConverge() {
return converge;
}
public void setConverge(List<Integer> converge) {
this.converge = converge;
}
public List<String> getNotify_group() {
return notify_group;
}
public void setNotify_group(List<String> notify_group) {
this.notify_group = notify_group;
}
public List<Integer> getNotify_user() {
return notify_user;
}
public void setNotify_user(List<Integer> notify_user) {
this.notify_user = notify_user;
}
public String getCallback() {
return callback;
}
public void setCallback(String callback) {
this.callback = callback;
}
public String getEnable_stime() {
return enable_stime;
}
public void setEnable_stime(String enable_stime) {
this.enable_stime = enable_stime;
}
public String getEnable_etime() {
return enable_etime;
}
public void setEnable_etime(String enable_etime) {
this.enable_etime = enable_etime;
}
public List<Integer> getEnable_days_of_week() {
return enable_days_of_week;
}
public void setEnable_days_of_week(List<Integer> enable_days_of_week) {
this.enable_days_of_week = enable_days_of_week;
}
public Integer getNeed_upgrade() {
return need_upgrade;
}
public void setNeed_upgrade(Integer need_upgrade) {
this.need_upgrade = need_upgrade;
}
@Override
public String toString() {
return "N9eStrategy{" +
"id=" + id +
", category=" + category +
", name='" + name + '\'' +
", nid=" + nid +
", excl_nid=" + excl_nid +
", priority=" + priority +
", alert_dur=" + alert_dur +
", exprs=" + exprs +
", tags=" + tags +
", recovery_dur=" + recovery_dur +
", recovery_notify=" + recovery_notify +
", alert_upgrade=" + alert_upgrade +
", converge=" + converge +
", notify_group=" + notify_group +
", notify_user=" + notify_user +
", callback='" + callback + '\'' +
", enable_stime='" + enable_stime + '\'' +
", enable_etime='" + enable_etime + '\'' +
", enable_days_of_week=" + enable_days_of_week +
", need_upgrade=" + need_upgrade +
'}';
}
}

59
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/entry/N9eStrategyAlertUpgrade.java Normal file
View File

@@ -0,0 +1,59 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e.entry;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/19
*/
public class N9eStrategyAlertUpgrade {
private Integer duration;
private Integer level;
private List<Integer> users;
private List<String> groups;
public Integer getDuration() {
return duration;
}
public void setDuration(Integer duration) {
this.duration = duration;
}
public Integer getLevel() {
return level;
}
public void setLevel(Integer level) {
this.level = level;
}
public List<Integer> getUsers() {
return users;
}
public void setUsers(List<Integer> users) {
this.users = users;
}
public List<String> getGroups() {
return groups;
}
public void setGroups(List<String> groups) {
this.groups = groups;
}
@Override
public String toString() {
return "N9eStrategyAlertUpgrade{" +
"duration=" + duration +
", level=" + level +
", users=" + users +
", groups=" + groups +
'}';
}
}

70
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/entry/N9eStrategyExpression.java Normal file
View File

@@ -0,0 +1,70 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e.entry;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/18
*/
public class N9eStrategyExpression {
private String metric;
private String func;
private String eopt;
private Integer threshold;
private List<Integer> params;
public String getMetric() {
return metric;
}
public void setMetric(String metric) {
this.metric = metric;
}
public String getFunc() {
return func;
}
public void setFunc(String func) {
this.func = func;
}
public String getEopt() {
return eopt;
}
public void setEopt(String eopt) {
this.eopt = eopt;
}
public Integer getThreshold() {
return threshold;
}
public void setThreshold(Integer threshold) {
this.threshold = threshold;
}
public List<Integer> getParams() {
return params;
}
public void setParams(List<Integer> params) {
this.params = params;
}
@Override
public String toString() {
return "N9eStrategyExpression{" +
"metric='" + metric + '\'' +
", func='" + func + '\'' +
", eopt='" + eopt + '\'' +
", threshold=" + threshold +
", params=" + params +
'}';
}
}

48
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/entry/N9eStrategyFilter.java Normal file
View File

@@ -0,0 +1,48 @@
package com.xiaojukeji.kafka.manager.monitor.component.n9e.entry;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/18
*/
public class N9eStrategyFilter {
private String topt;
private String tkey;
private List<String> tval;
public String getTopt() {
return topt;
}
public void setTopt(String topt) {
this.topt = topt;
}
public String getTkey() {
return tkey;
}
public void setTkey(String tkey) {
this.tkey = tkey;
}
public List<String> getTval() {
return tval;
}
public void setTval(List<String> tval) {
this.tval = tval;
}
@Override
public String toString() {
return "N9eStrategyFilter{" +
"topt='" + topt + '\'' +
", tkey='" + tkey + '\'' +
", tval=" + tval +
'}';
}
}

3
kafka-manager-extends/kafka-manager-monitor/src/main/resources/application-monitor-dev.yml
View File

@@ -1,3 +0,0 @@
monitor:
n9e:
base-url: http://127.0.0.1/api

4
kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/notifyer/KafkaNotifierService.java
View File

@@ -10,10 +10,10 @@ import org.springframework.stereotype.Service;
*/
@Service("notifyService")
public class KafkaNotifierService extends AbstractNotifyService {
@Value("${kafka.cluster-id:}")
@Value("${notify.kafka.cluster-id:}")
private Long clusterId;
@Value("${notify.topic-name:}")
@Value("${notify.kafka.topic-name:}")
private String topicName;
@Override

8
kafka-manager-extends/kafka-manager-notify/src/main/resources/application-notify-dev.yml
View File

@@ -1,8 +0,0 @@
notify:
order:
detail-url: http://127.0.0.1
kafka:
cluster-id: 12
topic-name: 123

20
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/ThirdPartService.java Normal file
View File

@@ -0,0 +1,20 @@
package com.xiaojukeji.kafka.manager.openapi;
import com.xiaojukeji.kafka.manager.common.bizenum.ConsumeHealthEnum;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.openapi.common.dto.*;
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
import java.util.List;
/**
* @author zengqiao
* @date 20/5/22
*/
public interface ThirdPartService {
Result<ConsumeHealthEnum> checkConsumeHealth(Long clusterId,
String topicName,
String consumerGroup,
Long maxDelayTime);
List<Result> resetOffsets(ClusterDO clusterDO, OffsetResetDTO dto);}

15
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/ThirdPartUtils.java Normal file
View File

@@ -0,0 +1,15 @@
package com.xiaojukeji.kafka.manager.openapi;
import com.xiaojukeji.kafka.manager.bpm.common.OrderTypeEnum;
/**
* @author zhongyuankai
* @date 2020/08/31
*/
public class ThirdPartUtils {
public static String getOrderLimitKey(OrderTypeEnum orderTypeEnum, String systemCode) {
return orderTypeEnum.getOrderName() + "_" + systemCode;
}
}

18
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/constant/ThirdPartConstant.java Normal file
View File

@@ -0,0 +1,18 @@
package com.xiaojukeji.kafka.manager.openapi.common.constant;
import java.util.Arrays;
import java.util.List;
/**
* @author zengqiao
* @date 20/10/26
*/
public class ThirdPartConstant {
public final static List<Long> QUOTA_MODIFY_WHITE_CLUSTER_LIST = Arrays.asList(70L, 46L);
public final static Integer DATA_DREAM_MAX_APP_NUM = 20;
public final static Integer DATA_DREAM_MAX_AUTHORITY_NUM = 500;
public final static String SELF_SYSTEM_CODE = "kafkamanager";
}

83
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/ConsumeHealthDTO.java Normal file
View File

@@ -0,0 +1,83 @@
package com.xiaojukeji.kafka.manager.openapi.common.dto;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List;
/**
* @author zengqiao
* @date 20/6/2
*/
@ApiModel(description = "消费健康")
public class ConsumeHealthDTO {
@ApiModelProperty(value = "集群ID")
private Long clusterId;
@ApiModelProperty(value = "Topic名称")
private List<String> topicNameList;
@ApiModelProperty(value = "消费组")
private String consumerGroup;
@ApiModelProperty(value = "允许最大延迟(ms)")
private Long maxDelayTime;
public Long getClusterId() {
return clusterId;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
public List<String> getTopicNameList() {
return topicNameList;
}
public void setTopicNameList(List<String> topicNameList) {
this.topicNameList = topicNameList;
}
public String getConsumerGroup() {
return consumerGroup;
}
public void setConsumerGroup(String consumerGroup) {
this.consumerGroup = consumerGroup;
}
public Long getMaxDelayTime() {
return maxDelayTime;
}
public void setMaxDelayTime(Long maxDelayTime) {
this.maxDelayTime = maxDelayTime;
}
@Override
public String toString() {
return "ConsumeHealthDTO{" +
"clusterId=" + clusterId +
", topicNameList=" + topicNameList +
", consumerGroup='" + consumerGroup + '\'' +
", maxDelayTime=" + maxDelayTime +
'}';
}
public boolean paramLegal() {
if (ValidateUtils.isNull(clusterId)
|| ValidateUtils.isEmptyList(topicNameList)
|| ValidateUtils.isBlank(consumerGroup)
|| ValidateUtils.isNullOrLessThanZero(maxDelayTime)) {
return false;
}
for (String topicName: topicNameList) {
if (ValidateUtils.isExistBlank(topicName)) {
return false;
}
}
return true;
}
}

208
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/dto/OffsetResetDTO.java Normal file
View File

@@ -0,0 +1,208 @@
package com.xiaojukeji.kafka.manager.openapi.common.dto;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.xiaojukeji.kafka.manager.common.bizenum.OffsetResetTypeEnum;
import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.springframework.util.StringUtils;
import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true)
@ApiModel(description = "重置消费偏移")
public class OffsetResetDTO {
@ApiModelProperty(value = "集群ID")
private Long clusterId;
@ApiModelProperty(value = "Topic名称")
private String topicName;
@ApiModelProperty(value = "消费组")
private String consumerGroup;
@ApiModelProperty(value = "消费组位置")
private String location;
@ApiModelProperty(value = "重置的方式[0:依据时间进行重置, 1:指定分区offset进行重置]")
private Integer offsetResetType;
@ApiModelProperty(value = "依据时间进行重置时, 传的参数, 13位时间戳")
private Long timestamp;
@ApiModelProperty(value = "指定分区进行重置时, 传的参数")
private List<PartitionOffsetDTO> partitionOffsetDTOList;
@ApiModelProperty(value = "如果消费组不存在则创建")
private Boolean createIfAbsent = Boolean.FALSE;
@ApiModelProperty(value = "使用的AppID")
private String appId;
@ApiModelProperty(value = "App密码")
private String password;
@ApiModelProperty(value = "操作人")
private String operator;
@ApiModelProperty(value = "系统code")
private String systemCode;
/**
* 默认使用assign的方式进行重置,
* 但是使用assign方式对于多个Topic的消费使用同一个消费组的场景, 需要停掉所有的client才可以重置成功, 否则重置失败
*
* 使用subscribe重置offset, 针对上面的场景可以重置成功, 但是涉及到poll函数调用, 所以默认是关闭的
*/
private Boolean subscribeReset = Boolean.FALSE; // 订阅重置, 默认是assign方式重置
public Long getClusterId() {
return clusterId;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
public String getTopicName() {
return topicName;
}
public void setTopicName(String topicName) {
this.topicName = topicName;
}
public String getConsumerGroup() {
return consumerGroup;
}
public void setConsumerGroup(String consumerGroup) {
this.consumerGroup = consumerGroup;
}
public String getLocation() {
return location;
}
public void setLocation(String location) {
this.location = location;
}
public Integer getOffsetResetType() {
return offsetResetType;
}
public void setOffsetResetType(Integer offsetResetType) {
this.offsetResetType = offsetResetType;
}
public Long getTimestamp() {
return timestamp;
}
public void setTimestamp(Long timestamp) {
this.timestamp = timestamp;
}
public List<PartitionOffsetDTO> getPartitionOffsetDTOList() {
return partitionOffsetDTOList;
}
public void setPartitionOffsetDTOList(List<PartitionOffsetDTO> partitionOffsetDTOList) {
this.partitionOffsetDTOList = partitionOffsetDTOList;
}
public Boolean getCreateIfAbsent() {
return createIfAbsent;
}
public void setCreateIfAbsent(Boolean createIfAbsent) {
this.createIfAbsent = createIfAbsent;
}
public String getAppId() {
return appId;
}
public void setAppId(String appId) {
this.appId = appId;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getOperator() {
return operator;
}
public void setOperator(String operator) {
this.operator = operator;
}
public String getSystemCode() {
return systemCode;
}
public void setSystemCode(String systemCode) {
this.systemCode = systemCode;
}
public Boolean getSubscribeReset() {
return subscribeReset;
}
public void setSubscribeReset(Boolean subscribeReset) {
this.subscribeReset = subscribeReset;
}
@Override
public String toString() {
return "OffsetResetModel{" +
"clusterId=" + clusterId +
", topicName='" + topicName + '\'' +
", consumerGroup='" + consumerGroup + '\'' +
", location='" + location + '\'' +
", offsetResetType=" + offsetResetType +
", timestamp=" + timestamp +
", partitionOffsetDTOList=" + partitionOffsetDTOList +
", createIfAbsent=" + createIfAbsent +
", appId='" + appId + '\'' +
", password='" + password + '\'' +
", operator='" + operator + '\'' +
", systemCode='" + systemCode + '\'' +
", subscribeReset=" + subscribeReset +
'}';
}
public boolean legal() {
if (clusterId == null
|| StringUtils.isEmpty(topicName)
|| StringUtils.isEmpty(consumerGroup)
|| StringUtils.isEmpty(location)
|| offsetResetType == null
|| StringUtils.isEmpty(operator)) {
return false;
}
appId = (appId == null? "": appId);
password = (password == null? "": password);
if (createIfAbsent == null) {
createIfAbsent = false;
}
if (subscribeReset == null) {
subscribeReset = false;
}
// 只能依据时间或者offset中的一个进行重置
if (OffsetResetTypeEnum.RESET_BY_TIME.getCode().equals(offsetResetType)) {
return timestamp != null;
} else if (OffsetResetTypeEnum.RESET_BY_OFFSET.getCode().equals(offsetResetType)) {
return partitionOffsetDTOList != null;
}
return false;
}
}

57
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/vo/BrokerRegionVO.java Normal file
View File

@@ -0,0 +1,57 @@
package com.xiaojukeji.kafka.manager.openapi.common.vo;
/**
* @author zengqiao
* @date 20/9/14
*/
public class BrokerRegionVO {
private Long clusterId;
private Integer brokerId;
private String hostname;
private String regionName;
public Long getClusterId() {
return clusterId;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
public Integer getBrokerId() {
return brokerId;
}
public void setBrokerId(Integer brokerId) {
this.brokerId = brokerId;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public String getRegionName() {
return regionName;
}
public void setRegionName(String regionName) {
this.regionName = regionName;
}
@Override
public String toString() {
return "BrokerRegionVO{" +
"clusterId=" + clusterId +
", brokerId=" + brokerId +
", hostname='" + hostname + '\'' +
", regionName='" + regionName + '\'' +
'}';
}
}

28
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/vo/ConsumeHealthVO.java Normal file
View File

@@ -0,0 +1,28 @@
package com.xiaojukeji.kafka.manager.openapi.common.vo;
/**
* @author zengqiao
* @date 20/10/26
*/
public class ConsumeHealthVO {
private Integer healthCode;
public ConsumeHealthVO(Integer healthCode) {
this.healthCode = healthCode;
}
public Integer getHealthCode() {
return healthCode;
}
public void setHealthCode(Integer healthCode) {
this.healthCode = healthCode;
}
@Override
public String toString() {
return "ConsumeHealthVO{" +
"healthCode=" + healthCode +
'}';
}
}

59
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/vo/ThirdPartBrokerOverviewVO.java Normal file
View File

@@ -0,0 +1,59 @@
package com.xiaojukeji.kafka.manager.openapi.common.vo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**
* @author zengqiao
* @date 20/9/9
*/
@ApiModel(description="第三方-Broker概览")
public class ThirdPartBrokerOverviewVO {
@ApiModelProperty(value = "集群ID")
private Long clusterId;
@ApiModelProperty(value = "BrokerId")
private Integer brokerId;
@ApiModelProperty(value = "处于同步状态 false:已同步, true:未同步")
private Boolean underReplicated;
public ThirdPartBrokerOverviewVO(Long clusterId, Integer brokerId, Boolean underReplicated) {
this.clusterId = clusterId;
this.brokerId = brokerId;
this.underReplicated = underReplicated;
}
public Long getClusterId() {
return clusterId;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
public Integer getBrokerId() {
return brokerId;
}
public void setBrokerId(Integer brokerId) {
this.brokerId = brokerId;
}
public Boolean getUnderReplicated() {
return underReplicated;
}
public void setUnderReplicated(Boolean underReplicated) {
this.underReplicated = underReplicated;
}
@Override
public String toString() {
return "ThirdPartBrokerOverviewVO{" +
"clusterId=" + clusterId +
", brokerId=" + brokerId +
", underReplicated=" + underReplicated +
'}';
}
}

33
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/vo/TopicOffsetChangedVO.java Normal file
View File

@@ -0,0 +1,33 @@
package com.xiaojukeji.kafka.manager.openapi.common.vo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**
* @author zengqiao
* @date 20/8/24
*/
@ApiModel(description="TopicOffset变化")
public class TopicOffsetChangedVO {
@ApiModelProperty(value="Offset是否变化, 0:否, 1:是, -1:未知")
private Integer offsetChanged;
public TopicOffsetChangedVO(Integer offsetChanged) {
this.offsetChanged = offsetChanged;
}
public Integer getOffsetChanged() {
return offsetChanged;
}
public void setOffsetChanged(Integer offsetChanged) {
this.offsetChanged = offsetChanged;
}
@Override
public String toString() {
return "TopicOffsetChangedVO{" +
"offsetChanged=" + offsetChanged +
'}';
}
}

34
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/common/vo/TopicStatisticMetricsVO.java Normal file
View File

@@ -0,0 +1,34 @@
package com.xiaojukeji.kafka.manager.openapi.common.vo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**
* @author zengqiao
* @date 20/8/14
*/
@ApiModel(description="Topic流量统计信息")
public class TopicStatisticMetricsVO {
@ApiModelProperty(value="峰值流入流量(B/s)")
private Double peakBytesIn;
public TopicStatisticMetricsVO(Double peakBytesIn) {
this.peakBytesIn = peakBytesIn;
}
public Double getPeakBytesIn() {
return peakBytesIn;
}
public void setPeakBytesIn(Double peakBytesIn) {
this.peakBytesIn = peakBytesIn;
}
@Override
public String toString() {
return "TopicStatisticMetricsVO{" +
"peakBytesIn=" + peakBytesIn +
'}';
}
}

200
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/impl/ThirdPartServiceImpl.java Normal file
View File

@@ -0,0 +1,200 @@
package com.xiaojukeji.kafka.manager.openapi.impl;
import com.xiaojukeji.kafka.manager.common.bizenum.*;
import com.xiaojukeji.kafka.manager.common.entity.Result;
import com.xiaojukeji.kafka.manager.common.entity.ResultStatus;
import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO;
import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata;
import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO;
import com.xiaojukeji.kafka.manager.openapi.ThirdPartService;
import com.xiaojukeji.kafka.manager.openapi.common.dto.*;
import com.xiaojukeji.kafka.manager.service.cache.KafkaClientPool;
import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager;
import com.xiaojukeji.kafka.manager.service.service.*;
import kafka.admin.AdminClient;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import scala.collection.JavaConversions;
import java.util.*;
/**
* @author zengqiao
* @date 20/5/22
*/
@Service("thirdPartService")
public class ThirdPartServiceImpl implements ThirdPartService {
private static Logger LOGGER = LoggerFactory.getLogger(ThirdPartServiceImpl.class);
@Autowired
private ClusterService clusterService;
@Autowired
private TopicService topicService;
@Autowired
private ConsumerService consumerService;
@Override
public Result<ConsumeHealthEnum> checkConsumeHealth(Long clusterId,
String topicName,
String consumerGroup,
Long maxDelayTime) {
ClusterDO clusterDO = clusterService.getById(clusterId);
if (ValidateUtils.isNull(clusterDO)) {
return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
}
TopicMetadata topicMetadata = PhysicalClusterMetadataManager.getTopicMetadata(clusterId, topicName);
if (ValidateUtils.isNull(topicMetadata)) {
return Result.buildFrom(ResultStatus.TOPIC_NOT_EXIST);
}
// 获取消费组当前的offset
Map<TopicPartition, Object> consumeOffsetMap = listGroupOffsets(clusterId, consumerGroup);
if (ValidateUtils.isNull(consumeOffsetMap)) {
return new Result<>(ConsumeHealthEnum.UNKNOWN);
}
if (consumeOffsetMap.isEmpty()) {
return Result.buildFrom(ResultStatus.CONSUMER_GROUP_NOT_EXIST);
}
Long delayTimestamp = System.currentTimeMillis() - maxDelayTime;
// 获取指定时间的offset
Map<TopicPartition, OffsetAndTimestamp> offsetAndTimeMap =
offsetsForTimes(clusterDO, topicMetadata, delayTimestamp);
if (ValidateUtils.isNull(offsetAndTimeMap)) {
return new Result<>(ConsumeHealthEnum.UNKNOWN);
}
for (TopicPartition tp : offsetAndTimeMap.keySet()) {
OffsetAndTimestamp offsetAndTimestamp = offsetAndTimeMap.get(tp);
Long consumeOffset = (Long) consumeOffsetMap.get(tp);
if (ValidateUtils.isNull(consumeOffset)) {
return new Result<>(ConsumeHealthEnum.UNKNOWN);
}
if (offsetAndTimestamp.offset() <= consumeOffset) {
// 健康的
continue;
}
return new Result<>(ConsumeHealthEnum.UNHEALTH);
}
return new Result<>(ConsumeHealthEnum.HEALTH);
}
private Map<TopicPartition, Object> listGroupOffsets(Long clusterId, String consumerGroup) {
AdminClient client = KafkaClientPool.getAdminClient(clusterId);
if (ValidateUtils.isNull(client)) {
return null;
}
try {
return JavaConversions.asJavaMap(client.listGroupOffsets(consumerGroup));
} catch (Exception e) {
LOGGER.error("list group offsets failed, clusterId:{}, consumerGroup:{}.", clusterId, consumerGroup, e);
}
return null;
}
private Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(ClusterDO clusterDO,
TopicMetadata topicMetadata,
Long timestamp) {
KafkaConsumer kafkaConsumer = null;
try {
kafkaConsumer = KafkaClientPool.borrowKafkaConsumerClient(clusterDO);
if (ValidateUtils.isNull(kafkaConsumer)) {
return null;
}
Map<TopicPartition, Long> timestampsToSearch = new HashMap<>();
for (Integer partitionId : topicMetadata.getPartitionMap().getPartitions().keySet()) {
timestampsToSearch.put(new TopicPartition(topicMetadata.getTopic(), partitionId), timestamp);
}
return kafkaConsumer.offsetsForTimes(timestampsToSearch);
} catch (Exception e) {
LOGGER.error("get offset for time failed, clusterDO:{} topicMetadata:{} timestamp:{}.",
clusterDO, topicMetadata, timestamp, e);
} finally {
KafkaClientPool.returnKafkaConsumerClient(clusterDO.getId(), kafkaConsumer);
}
return null;
}
@Override
public List<Result> resetOffsets(ClusterDO clusterDO, OffsetResetDTO dto) {
if (ValidateUtils.isNull(dto)) {
return null;
}
List<PartitionOffsetDTO> offsetDTOList = dto.getPartitionOffsetDTOList();
if (ValidateUtils.isEmptyList(offsetDTOList)) {
offsetDTOList = topicService.getPartitionOffsetList(
clusterDO, dto.getTopicName(), dto.getTimestamp());
}
if (ValidateUtils.isEmptyList(offsetDTOList)) {
return null;
}
OffsetLocationEnum offsetLocation = dto.getLocation().equals(
OffsetLocationEnum.ZOOKEEPER.location) ? OffsetLocationEnum.ZOOKEEPER : OffsetLocationEnum.BROKER;
ResultStatus result = checkConsumerGroupExist(clusterDO, dto.getTopicName(), dto.getConsumerGroup(), offsetLocation, dto.getCreateIfAbsent());
if (ResultStatus.SUCCESS.getCode() != result.getCode()) {
return null;
}
ConsumerGroupDTO consumerGroupDTO = new ConsumerGroupDTO(
clusterDO.getId(),
dto.getConsumerGroup(),
new ArrayList<>(),
OffsetLocationEnum.getOffsetStoreLocation(dto.getLocation())
);
return consumerService.resetConsumerOffset(
clusterDO,
dto.getTopicName(),
consumerGroupDTO,
offsetDTOList
);
}
private ResultStatus checkConsumerGroupExist(ClusterDO clusterDO,
String topicName,
String consumerGroup,
OffsetLocationEnum offsetLocation,
Boolean createIfAbsent) {
if (createIfAbsent) {
// 如果不存在, 则直接创建
return isCreateIfAbsentOverflow(clusterDO, topicName);
}
if (!consumerService.checkConsumerGroupExist(offsetLocation, clusterDO.getId(), topicName, consumerGroup)) {
return ResultStatus.PARAM_ILLEGAL;
}
return ResultStatus.SUCCESS;
}
/**
* 限制单天单集群的重置次数不能超过20个
* <clusterId-topicName, timestamp * 100 + count>
*/
private static final Map<String, Long> createIfAbsentCountMap = new HashMap<>();
private synchronized ResultStatus isCreateIfAbsentOverflow(ClusterDO clusterDO, String topicName) {
String key = clusterDO.getId() + "_" + topicName;
Long timestampAndCount = createIfAbsentCountMap.get(key);
if (ValidateUtils.isNull(timestampAndCount) ||
(System.currentTimeMillis() - (timestampAndCount / 100) >= (24 *60 * 60 * 1000))) {
// 24小时卫触发, 统计归0
timestampAndCount = System.currentTimeMillis() * 100L + 1;
} else if (timestampAndCount % 100 > 20) {
return ResultStatus.OPERATION_FORBIDDEN;
} else {
timestampAndCount += 1;
}
createIfAbsentCountMap.put(key, timestampAndCount);
return ResultStatus.SUCCESS;
}
}