8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

初始化3.0.0版本

Browse Source
This commit is contained in:
zengqiao
2022-08-18 17:04:05 +08:00
parent 462303fca0
commit 51832385b1
2446 changed files with 93177 additions and 127211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
km-extends/km-account/pom.xml Normal file
View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-account</artifactId>
<version>${km.revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencies>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-core</artifactId>
<version>${project.parent.version}</version>
</dependency>
</dependencies>
</project>

53
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/KmAccountConfig.java Normal file
View File

@@ -0,0 +1,53 @@
package com.xiaojukeji.know.streaming.km.account;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Data
@Service
public class KmAccountConfig {
/**
* LoginService的默认配置
*/
@Value(value = "${account.login.service.name:loginService}")
private String loginServiceName;
/**************************************************** Ldap 登录相关配置 ****************************************************/
@Value(value = "${account.ldap.url:}")
private String ldapUrl;
@Value(value = "${account.ldap.basedn:}")
private String ldapBaseDN;
@Value(value = "${account.ldap.factory:}")
private String ldapFactory;
@Value(value = "${account.ldap.filter:}")
private String ldapFilter;
@Value(value = "${account.ldap.security.authentication:}")
private String securityAuthentication;
@Value(value = "${account.ldap.security.principal:}")
private String securityPrincipal;
@Value(value = "${account.ldap.security.credentials:}")
private String securityCredentials;
/**************************************************** Ldap 用户注册到KM相关 ****************************************************/
@Value(value = "${account.ldap.auth-user-registration-role:0,1,2}")
private String authUserRegistrationRole; // ldap自动注册的默认角色。请注意：它通常来说都是低权限角色
@Value(value = "${account.ldap.auth-user-registration:false}")
private Boolean authUserRegistration; // ldap自动注册是否开启
private KmAccountConfig() {
}
}

25
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/common/bizenum/LoginServiceNameEnum.java Normal file
View File

@@ -0,0 +1,25 @@
package com.xiaojukeji.know.streaming.km.account.common.bizenum;
import lombok.Getter;
@Getter
public enum LoginServiceNameEnum {
DEFAULT(LoginServiceNameEnum.DEFAULT_LOGIN_NAME, "默认"),
LDAP(LoginServiceNameEnum.LDAP_LOGIN_NAME, "Ldap登录"),
;
public static final String DEFAULT_LOGIN_NAME = "loginService";
public static final String LDAP_LOGIN_NAME = "ldapLoginService";
private final String name;
private final String msg;
LoginServiceNameEnum(String name, String msg) {
this.name = name;
this.msg = msg;
}
}

24
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/common/constant/TrickJumpLoginConstant.java Normal file
View File

@@ -0,0 +1,24 @@
package com.xiaojukeji.know.streaming.km.account.common.constant;
public class TrickJumpLoginConstant {
/**
* HTTP Header key
*/
public static final String TRICK_JUMP_LOGIN_SWITCH = "Trick-Login-Switch";
public static final String TRICK_JUMP_LOGIN_USER = "Trick-Login-User";
/**
* 配置允许 trick 登录用户名单
*/
public static final String TRICK_JUMP_LOGIN_LEGAL_USER_CONFIG_GROUP = "SECURITY.LOGIN";
public static final String TRICK_JUMP_LOGIN_LEGAL_USER_CONFIG_NAME = "SECURITY.TRICK_USERS";
/**
* 开关状态值
*/
public static final String TRICK_JUMP_LOGIN_SWITCH_ON = "on";
private TrickJumpLoginConstant() {
}
}

18
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/common/ldap/LdapPrincipal.java Normal file
View File

@@ -0,0 +1,18 @@
package com.xiaojukeji.know.streaming.km.account.common.ldap;
import lombok.Data;
@Data
public class LdapPrincipal {
private String userDN;
private String sAMAccountName;
private String department;
private String company;
private String displayName;
private String mail;
}

16
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/common/ldap/exception/LdapException.java Normal file
View File

@@ -0,0 +1,16 @@
package com.xiaojukeji.know.streaming.km.account.common.ldap.exception;
import com.didiglobal.logi.security.exception.CodeMsg;
public class LdapException extends RuntimeException {
public LdapException() {}
public LdapException(CodeMsg codeMsg) {
super(codeMsg.getCode() + "-" + codeMsg.getMessage());
}
public LdapException(String message, Throwable cause) {
super(message, cause);
}
}

147
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/login/ldap/LdapLoginServiceImpl.java Normal file
View File

@@ -0,0 +1,147 @@
package com.xiaojukeji.know.streaming.km.account.login.ldap;
import com.didiglobal.logi.security.common.Result;
import com.didiglobal.logi.security.common.dto.account.AccountLoginDTO;
import com.didiglobal.logi.security.common.dto.user.UserDTO;
import com.didiglobal.logi.security.common.entity.user.User;
import com.didiglobal.logi.security.common.enums.ResultCode;
import com.didiglobal.logi.security.common.vo.user.UserBriefVO;
import com.didiglobal.logi.security.exception.LogiSecurityException;
import com.didiglobal.logi.security.service.LoginService;
import com.didiglobal.logi.security.service.UserService;
import com.didiglobal.logi.security.util.AESUtils;
import com.didiglobal.logi.security.util.CopyBeanUtil;
import com.didiglobal.logi.security.util.HttpRequestUtil;
import com.xiaojukeji.know.streaming.km.account.KmAccountConfig;
import com.xiaojukeji.know.streaming.km.account.common.bizenum.LoginServiceNameEnum;
import com.xiaojukeji.know.streaming.km.account.common.ldap.LdapPrincipal;
import com.xiaojukeji.know.streaming.km.account.login.ldap.remote.LdapAuthentication;
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import java.io.IOException;
import java.util.List;
import static com.didiglobal.logi.security.util.HttpRequestUtil.*;
import static com.didiglobal.logi.security.util.HttpRequestUtil.COOKIE_OR_SESSION_MAX_AGE_UNIT_SEC;
/**
* @author Hu.Yue
* @date 2021/8/4
*/
//@Service(LoginServiceNameEnum.LDAP_LOGIN_NAME)
public class LdapLoginServiceImpl implements LoginService {
private static final Logger LOGGER = LoggerFactory.getLogger(LdapLoginServiceImpl.class);
@Autowired
private UserService userService;
@Autowired
private KmAccountConfig kmAccountConfig;
@Autowired
private LdapAuthentication ldapAuthentication;
@Override
public UserBriefVO verifyLogin(AccountLoginDTO loginDTO,
HttpServletRequest request,
HttpServletResponse response) throws LogiSecurityException {
String decodePasswd = AESUtils.decrypt(loginDTO.getPw());
// 去LDAP验证账密
LdapPrincipal ldapAttrsInfo = ldapAuthentication.authenticate(loginDTO.getUserName(), decodePasswd);
if(ldapAttrsInfo == null) {
// 用户不存在，正常来说上如果有问题，上一步会直接抛出异常
throw new LogiSecurityException(ResultCode.USER_NOT_EXISTS);
}
// 如果用户不存在，则进行插入
User user = userService.getUserByUserName(ldapAttrsInfo.getSAMAccountName());
if(ValidateUtils.isNull(user) && kmAccountConfig.getAuthUserRegistration() != null && kmAccountConfig.getAuthUserRegistration()) {
// 自动注册用户
UserDTO userDTO = new UserDTO();
userDTO.setUserName(ldapAttrsInfo.getSAMAccountName());
userDTO.setPw(decodePasswd);
userDTO.setRealName(ldapAttrsInfo.getDisplayName());
userDTO.setPhone("");
userDTO.setEmail(ldapAttrsInfo.getMail());
userDTO.setRoleIds(CommonUtils.string2IntList(kmAccountConfig.getAuthUserRegistrationRole()));
userService.addUser(userDTO, ldapAttrsInfo.getSAMAccountName());
// user赋值
user = ConvertUtil.obj2Obj(userDTO, User.class);
}
// 记录登录状态
initLoginContext(request, response, loginDTO.getUserName(), user.getId());
return CopyBeanUtil.copy(user, UserBriefVO.class);
}
@Override
public Result<Boolean> logout(HttpServletRequest request, HttpServletResponse response){
request.getSession().invalidate();
response.setStatus(REDIRECT_CODE);
return Result.buildSucc(Boolean.TRUE);
}
@Override
public boolean interceptorCheck(HttpServletRequest request, HttpServletResponse response, String requestMappingValue, List<String> whiteMappingValues) throws IOException {
if (StringUtils.isEmpty(requestMappingValue)) {
LOGGER.error("method=interceptorCheck||msg=uri illegal||uri={}", request.getRequestURI());
return Boolean.FALSE;
}
// 白名单接口
for(String mapping : whiteMappingValues){
if (requestMappingValue.contains(mapping)){
return Boolean.TRUE;
}
}
// 检查是否已经登录
String userName = HttpRequestUtil.getOperator(request);
if(StringUtils.isEmpty(userName)) {
// 未登录
logout(request, response);
return Boolean.FALSE;
}
// 检查用户是否存在
User user = userService.getUserByUserName(userName);
if(user == null) {
throw new LogiSecurityException(ResultCode.USER_NOT_EXISTS);
}
initLoginContext(request, response, userName, user.getId());
return Boolean.TRUE;
}
private void initLoginContext(HttpServletRequest request, HttpServletResponse response, String userName, Integer userId) {
HttpSession session = request.getSession(true);
session.setMaxInactiveInterval( COOKIE_OR_SESSION_MAX_AGE_UNIT_SEC );
session.setAttribute(USER, userName);
session.setAttribute(USER_ID, userId);
Cookie cookieUserName = new Cookie(USER, userName);
cookieUserName.setMaxAge(COOKIE_OR_SESSION_MAX_AGE_UNIT_SEC);
cookieUserName.setPath("/");
Cookie cookieUserId = new Cookie(USER_ID, userId.toString());
cookieUserId.setMaxAge(COOKIE_OR_SESSION_MAX_AGE_UNIT_SEC);
cookieUserId.setPath("/");
response.addCookie(cookieUserName);
response.addCookie(cookieUserId);
}
}

144
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/login/ldap/remote/LdapAuthentication.java Normal file
View File

@@ -0,0 +1,144 @@
package com.xiaojukeji.know.streaming.km.account.login.ldap.remote;
import com.didiglobal.logi.security.common.enums.ResultCode;
import com.didiglobal.logi.security.exception.LogiSecurityException;
import com.xiaojukeji.know.streaming.km.account.KmAccountConfig;
import com.xiaojukeji.know.streaming.km.account.common.ldap.LdapPrincipal;
import com.xiaojukeji.know.streaming.km.account.common.ldap.exception.LdapException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.naming.Context;
import javax.naming.NamingEnumeration;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.InitialLdapContext;
import javax.naming.ldap.LdapContext;
import java.util.Hashtable;
/**
* @author Hu.Yue
* @date 2021/8/4
**/
@Component
public class LdapAuthentication {
private static final Logger LOGGER = LoggerFactory.getLogger(LdapAuthentication.class);
@Autowired
private KmAccountConfig kmAccountConfig;
/**
* LDAP账密验证
*/
public LdapPrincipal authenticate(String userName, String password) throws LogiSecurityException, LdapException {
// 获取ldap-context
LdapContext ctx = getLdapContext();
// 获取用户信息
LdapPrincipal ldapAttrsInfo = getLdapAttrsInfo(userName, ctx);
// 校验密码
try {
// 校验密码
ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapAttrsInfo.getUserDN());
ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password);
ctx.reconnect(null);
// 返回用户信息
return ldapAttrsInfo;
} catch (Exception e) {
LOGGER.error("method=authenticate||userName={}||errMsg={}", userName, e);
// 密码错误
throw new LogiSecurityException(ResultCode.USER_CREDENTIALS_ERROR);
} finally {
this.closeLdapContext(ctx);
}
}
/**************************************************** private method ****************************************************/
private LdapContext getLdapContext() throws LdapException {
Hashtable<String, String> env = new Hashtable<>();
env.put(Context.INITIAL_CONTEXT_FACTORY, kmAccountConfig.getLdapFactory());
env.put(Context.PROVIDER_URL, kmAccountConfig.getLdapUrl() + kmAccountConfig.getLdapBaseDN());
env.put(Context.SECURITY_AUTHENTICATION, kmAccountConfig.getSecurityAuthentication());
// 此处若不指定用户名和密码, 则自动转换为匿名登录
env.put(Context.SECURITY_PRINCIPAL, kmAccountConfig.getSecurityPrincipal());
env.put(Context.SECURITY_CREDENTIALS, kmAccountConfig.getSecurityCredentials());
try {
return new InitialLdapContext(env, null);
} catch (Exception e) {
LOGGER.error("method=getLdapContext||errMsg=exception", e);
throw new LdapException("调用Ldap服务异常", e);
}
}
/**
* 获取用户信息
*/
private LdapPrincipal getLdapAttrsInfo(String userName, LdapContext ctx) {
//存储更多的LDAP元信息
try {
SearchControls constraints = new SearchControls();
constraints.setSearchScope(SearchControls.SUBTREE_SCOPE);
// 查找
NamingEnumeration<SearchResult> en = ctx.search(
"",
"(&(objectClass=*)(" + kmAccountConfig.getLdapFilter() + "=" + userName + "))",
constraints
);
if (en == null || !en.hasMoreElements()) {
// 用户不存在
throw new LogiSecurityException(ResultCode.USER_NOT_EXISTS);
}
// maybe more than one element
while (en.hasMoreElements()) {
Object obj = en.nextElement();
if (obj instanceof SearchResult) {
SearchResult si = (SearchResult) obj;
// 携带LDAP更多元信息以填充用户元信息
LdapPrincipal ldapPrincipal = new LdapPrincipal();
ldapPrincipal.setUserDN(si.getName() + "," + kmAccountConfig.getLdapBaseDN());
ldapPrincipal.setSAMAccountName(this.keyValueSplit(si.getAttributes().get("samaccountname").toString()));
ldapPrincipal.setDepartment(this.keyValueSplit(si.getAttributes().get("department").toString()));
ldapPrincipal.setCompany(this.keyValueSplit(si.getAttributes().get("company").toString()));
ldapPrincipal.setDisplayName(this.keyValueSplit(si.getAttributes().get("displayname").toString()));
ldapPrincipal.setMail(this.keyValueSplit(si.getAttributes().get("mail").toString()));
return ldapPrincipal;
}
}
// 用户不存在
throw new LogiSecurityException(ResultCode.USER_NOT_EXISTS);
} catch (Exception e) {
LOGGER.error("method=getLdapAttrsInfo||userName={}||errMsg=exception", userName, e);
throw new LdapException("调用Ldap服务异常", e);
}
}
private void closeLdapContext(LdapContext ctx) {
if (ctx == null) {
return;
}
try {
ctx.close();
} catch (Exception e) {
LOGGER.error("method=closeLdapContext||errMsg=exception", e);
}
}
public String keyValueSplit(String keyValue){
return keyValue.split(":\\s+")[1];
}
}

19
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/login/trick/TrickJumpLoginService.java Normal file
View File

@@ -0,0 +1,19 @@
package com.xiaojukeji.know.streaming.km.account.login.trick;
import javax.servlet.http.HttpServletRequest;
/**
* @author zengqiao
* @date 21/5/18
*/
public interface TrickJumpLoginService {
/**
* 是否开启trick的方式登录
*/
boolean isOpenTrickJumpLogin(HttpServletRequest request);
/**
* 开启trick方式登录后，当前用户是否可以登录
*/
String checkTrickJumpLogin(HttpServletRequest request);
}

71
km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/login/trick/impl/TrickJumpLoginServiceImpl.java Normal file
View File

@@ -0,0 +1,71 @@
package com.xiaojukeji.know.streaming.km.account.login.trick.impl;
import com.didiglobal.logi.security.common.vo.config.ConfigVO;
import com.didiglobal.logi.security.service.ConfigService;
import com.xiaojukeji.know.streaming.km.account.common.constant.TrickJumpLoginConstant;
import com.xiaojukeji.know.streaming.km.account.login.trick.TrickJumpLoginService;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author zengqiao
* @date 21/5/18
*/
@Service
public class TrickJumpLoginServiceImpl implements TrickJumpLoginService {
private static final Logger LOGGER = LoggerFactory.getLogger(TrickJumpLoginServiceImpl.class);
@Autowired
private ConfigService configService;
@Override
public boolean isOpenTrickJumpLogin(HttpServletRequest request) {
return TrickJumpLoginConstant.TRICK_JUMP_LOGIN_SWITCH_ON.equals(request.getHeader(TrickJumpLoginConstant.TRICK_JUMP_LOGIN_SWITCH));
}
@Override
public String checkTrickJumpLogin(HttpServletRequest request) {
String trickLoginUser = request.getHeader(TrickJumpLoginConstant.TRICK_JUMP_LOGIN_USER);
LOGGER.info("method=checkTrickJumpLogin||userName={}||uri={}||msg=try trick jump login", trickLoginUser, request.getRequestURI());
if (!checkTrickJumpLogin(trickLoginUser)) {
LOGGER.warn("method=checkTrickJumpLogin||userName={}||uri={}||msg=trick login failed", trickLoginUser, request.getRequestURI());
return null;
}
return trickLoginUser;
}
private boolean checkTrickJumpLogin(String trickJumpLoginUser) {
List<ConfigVO> voList = configService.listConfigByGroup(TrickJumpLoginConstant.TRICK_JUMP_LOGIN_LEGAL_USER_CONFIG_GROUP)
.stream()
.filter(elem -> elem.getValueName().equals(TrickJumpLoginConstant.TRICK_JUMP_LOGIN_LEGAL_USER_CONFIG_NAME))
.collect(Collectors.toList());
if (ValidateUtils.isEmptyList(voList)) {
// 不存在
return false;
}
for (ConfigVO vo: voList) {
List<String> canJumpUserNameList = ConvertUtil.str2ObjArrayByJson(vo.getValue(), String.class);
if (ValidateUtils.isEmptyList(canJumpUserNameList)) {
continue;
}
if (canJumpUserNameList.contains(trickJumpLoginUser)) {
return true;
}
}
return false;
}
}

50
km-extends/km-monitor/pom.xml Normal file
View File

@@ -0,0 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-monitor</artifactId>
<version>${km.revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencies>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-persistence</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-core</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
</dependencies>
</project>

14
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/common/AbstractMonitorSinkTag.java Normal file
View File

@@ -0,0 +1,14 @@
package com.xiaojukeji.know.streaming.km.monitor.common;
import java.util.Map;
/**
* @author zengqiao
* @date 20/5/24
*/
public abstract class AbstractMonitorSinkTag {
public abstract String convert2Tags();
public abstract Map<String, Object> tagsMap();
}

35
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/common/MetricSinkPoint.java Normal file
View File

@@ -0,0 +1,35 @@
package com.xiaojukeji.know.streaming.km.monitor.common;
import lombok.Data;
import java.util.Map;
@Data
public class MetricSinkPoint {
/**
* 指标名
*/
private String name;
/**
* 指标值
*/
private Float value;
/**
* 上报周期，单位秒
*/
private int step;
/**
* 当前时间戳，单位为s
*/
private long timestamp;
/**
* tags
*/
private Map<String, Object> tagsMap;
public MetricSinkPoint() {}
}

30
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/common/MonitorSinkTagEnum.java Normal file
View File

@@ -0,0 +1,30 @@
package com.xiaojukeji.know.streaming.km.monitor.common;
public enum MonitorSinkTagEnum {
/**
* cluster
*/
CLUSTER_ID("clusterId"),
BROKER_ID("brokerId"),
TOPIC("topic"),
PARTITION_ID("partitionId"),
CONSUMER_GROUP("consumerGroup"),
REPLICATION("replication"),
;
private final String name;
MonitorSinkTagEnum(String name) {
this.name = name;
}
public String getName() {
return name;
}
}

180
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/component/AbstractMonitorSinkService.java Normal file
View File

@@ -0,0 +1,180 @@
package com.xiaojukeji.know.streaming.km.monitor.component;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.*;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.*;
import com.xiaojukeji.know.streaming.km.common.utils.NamedThreadFactory;
import com.xiaojukeji.know.streaming.km.monitor.common.MetricSinkPoint;
import org.springframework.context.ApplicationListener;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static com.xiaojukeji.know.streaming.km.monitor.common.MonitorSinkTagEnum.*;
public abstract class AbstractMonitorSinkService implements ApplicationListener<BaseMetricEvent> {
protected static final ILog LOGGER = LogFactory.getLog(AbstractMonitorSinkService.class);
private static final int STEP = 60;
private ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 10, 6000, TimeUnit.MILLISECONDS,
new LinkedBlockingDeque<>(1000),
new NamedThreadFactory("KM-Monitor-Sink-" + monitorName()),
(r, e) -> LOGGER.warn("class=AbstractMonitorSinkService||msg=Deque is blocked, taskCount:{}" + e.getTaskCount()));
/**
* monitor 服务的名称
* @return
*/
protected abstract String monitorName();
@Override
public void onApplicationEvent(BaseMetricEvent event) {
executor.execute( () -> {
if(event instanceof BrokerMetricEvent){
BrokerMetricEvent brokerMetricEvent = (BrokerMetricEvent)event;
sinkMetrics(brokerMetric2SinkPoint(brokerMetricEvent.getBrokerMetrics()));
}else if(event instanceof ClusterMetricEvent){
ClusterMetricEvent clusterMetricEvent = (ClusterMetricEvent)event;
sinkMetrics(clusterMetric2SinkPoint(clusterMetricEvent.getClusterMetrics()));
}else if(event instanceof TopicMetricEvent){
TopicMetricEvent topicMetricEvent = (TopicMetricEvent)event;
sinkMetrics(topicMetric2SinkPoint(topicMetricEvent.getTopicMetrics()));
}else if(event instanceof PartitionMetricEvent){
PartitionMetricEvent partitionMetricEvent = (PartitionMetricEvent)event;
sinkMetrics(partitionMetric2SinkPoint(partitionMetricEvent.getPartitionMetrics()));
}else if(event instanceof GroupMetricEvent){
GroupMetricEvent groupMetricEvent = (GroupMetricEvent)event;
sinkMetrics(groupMetric2SinkPoint(groupMetricEvent.getGroupMetrics()));
}else if(event instanceof ReplicaMetricEvent){
ReplicaMetricEvent replicaMetricEvent = (ReplicaMetricEvent)event;
sinkMetrics(replicationMetric2SinkPoint(replicaMetricEvent.getReplicationMetrics()));
}
} );
}
/**
* 监控指标的上报和查询
* @param pointList
* @return
*/
public abstract Boolean sinkMetrics(List<MetricSinkPoint> pointList);
/**************************************************** private method ****************************************************/
private List<MetricSinkPoint> brokerMetric2SinkPoint(List<BrokerMetrics> brokerMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(BrokerMetrics b : brokerMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), b.getClusterPhyId());
tagsMap.put(BROKER_ID.getName(), b.getBrokerId());
pointList.addAll(genSinkPoint("Broker", b.getMetrics(), b.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> clusterMetric2SinkPoint(List<ClusterMetrics> clusterMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(ClusterMetrics c : clusterMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), c.getClusterPhyId());
pointList.addAll(genSinkPoint("Cluster", c.getMetrics(), c.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> topicMetric2SinkPoint(List<TopicMetrics> topicMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(TopicMetrics t : topicMetrics){
if(t.isBBrokerAgg()){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), t.getClusterPhyId());
tagsMap.put(TOPIC.getName(), t.getTopic());
pointList.addAll(genSinkPoint("Topic", t.getMetrics(), t.getTimestamp(), tagsMap));
}
}
return pointList;
}
private List<MetricSinkPoint> partitionMetric2SinkPoint(List<PartitionMetrics> partitionMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(PartitionMetrics p : partitionMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), p.getClusterPhyId());
tagsMap.put(BROKER_ID.getName(), p.getBrokerId());
tagsMap.put(PARTITION_ID.getName(), p.getPartitionId());
pointList.addAll(genSinkPoint("Partition", p.getMetrics(), p.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> groupMetric2SinkPoint(List<GroupMetrics> groupMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(GroupMetrics g : groupMetrics){
if(g.isBGroupMetric()){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), g.getClusterPhyId());
tagsMap.put(CONSUMER_GROUP.getName(), g.getGroup());
pointList.addAll(genSinkPoint("Group", g.getMetrics(), g.getTimestamp(), tagsMap));
}
}
return pointList;
}
private List<MetricSinkPoint> replicationMetric2SinkPoint(List<ReplicationMetrics> replicationMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(ReplicationMetrics r : replicationMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), r.getClusterPhyId());
tagsMap.put(BROKER_ID.getName(), r.getBrokerId());
tagsMap.put(PARTITION_ID.getName(), r.getPartitionId());
pointList.addAll(genSinkPoint("Replication", r.getMetrics(), r.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> genSinkPoint(String metricPre, Map<String, Float> metrics,
long timeStamp, Map<String, Object> tagsMap){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(String metricName : metrics.keySet()){
MetricSinkPoint metricSinkPoint = new MetricSinkPoint();
metricSinkPoint.setName(metricPre + "_" + metricName);
metricSinkPoint.setValue(metrics.get(metricName));
metricSinkPoint.setTimestamp(timeStamp);
metricSinkPoint.setStep(STEP);
metricSinkPoint.setTagsMap(tagsMap);
pointList.add(metricSinkPoint);
}
return pointList;
}
}

85
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/component/prometheus/PrometheusService.java Normal file
View File

@@ -0,0 +1,85 @@
package com.xiaojukeji.know.streaming.km.monitor.component.prometheus;
import com.alibaba.fastjson.JSON;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.monitor.common.MetricSinkPoint;
import com.xiaojukeji.know.streaming.km.monitor.common.MonitorSinkTagEnum;
import com.xiaojukeji.know.streaming.km.monitor.component.AbstractMonitorSinkService;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.Gauge;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author didi
*/
@Service("prometheusService")
public class PrometheusService extends AbstractMonitorSinkService {
private static final ILog LOGGER = LogFactory.getLog(PrometheusService.class);
private final Map<String, Gauge> gaugeMap = new ConcurrentHashMap<>();
private final List<String> monitorSinkTagNames = new ArrayList<>();
@Autowired
public CollectorRegistry collectorRegistry;
@PostConstruct
public void init(){
for(MonitorSinkTagEnum monitorSinkTagEnum : MonitorSinkTagEnum.values()){
monitorSinkTagNames.add(monitorSinkTagEnum.getName());
}
monitorSinkTagNames.sort(Comparator.reverseOrder());
}
@Override
protected String monitorName() {
return "prometheus";
}
@Override
public Boolean sinkMetrics(List<MetricSinkPoint> pointList) {
if(ValidateUtils.isEmptyList(pointList)){
return true;
}
for (MetricSinkPoint metricSinkPoint : pointList){
try {
String metricName = metricSinkPoint.getName().replace("-", "_");
Gauge gauge = gaugeMap.get(metricName);
if(null == gauge) {
gauge = Gauge.build().name(metricName)
.help(metricName + " metrics")
.labelNames(monitorSinkTagNames.toArray(new String[monitorSinkTagNames.size()]))
.register(collectorRegistry);
gaugeMap.put(metricName, gauge);
}
Map<String, Object> tagsMap = metricSinkPoint.getTagsMap();
List<String> labels = new ArrayList<>();
for(String metricTagName : monitorSinkTagNames){
if (tagsMap.containsKey(metricTagName)){
labels.add(tagsMap.get(metricTagName).toString());
} else {
labels.add("");
}
}
gauge.labels(labels.toArray(new String[labels.size()])).set(metricSinkPoint.getValue());
} catch (Exception e){
LOGGER.error("sink metrics to prometheus exception, metric:{}", JSON.toJSONString(metricSinkPoint), e);
}
}
return true;
}
}