8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2026-01-05 13:08:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #473 from didi/dev

Browse Source 
Dev
This commit is contained in:
EricZeng
2022-03-07 14:49:52 +08:00
committed by GitHub
parent 482a375e31 44d537f78c
commit e51dbe0ca7
147 changed files with 1828 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/account/BaseEnterpriseStaffService.java
View File

@@ -54,7 +54,8 @@ public class BaseEnterpriseStaffService extends AbstractEnterpriseStaffService {
}
List<EnterpriseStaff> staffList = new ArrayList<>();
for (AccountDO accountDO: doList) {
staffList.add(new EnterpriseStaff(accountDO.getUsername(), accountDO.getUsername(), ""));
//这里对chineseName填充共识的displayName，Department则获取Department信息
staffList.add(new EnterpriseStaff(accountDO.getUsername(), accountDO.getDisplayName(), accountDO.getDepartment()));
}
return staffList;
} catch (Exception e) {

40
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/ldap/LdapAuthentication.java
View File

@@ -1,5 +1,6 @@
package com.xiaojukeji.kafka.manager.account.component.ldap;
import com.xiaojukeji.kafka.manager.common.utils.SplitUtils;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -14,7 +15,9 @@ import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.InitialLdapContext;
import javax.naming.ldap.LdapContext;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Map;
@Component
public class LdapAuthentication {
@@ -60,8 +63,11 @@ public class LdapAuthentication {
return null;
}
private String getUserDN(String account, LdapContext ctx) {
private Map<String, Object> getLdapAttrsInfo(String account, LdapContext ctx) {
//存储更多的LDAP元信息
Map<String, Object> ldapAttrsInfo = new HashMap<>();
String userDN = "";
ldapAttrsInfo.clear();
try {
SearchControls constraints = new SearchControls();
constraints.setSearchScope(SearchControls.SUBTREE_SCOPE);
@@ -69,7 +75,7 @@ public class LdapAuthentication {
NamingEnumeration<SearchResult> en = ctx.search("", filter, constraints);
if (en == null || !en.hasMoreElements()) {
return "";
return null;
}
// maybe more than one element
while (en.hasMoreElements()) {
@@ -78,13 +84,25 @@ public class LdapAuthentication {
SearchResult si = (SearchResult) obj;
userDN += si.getName();
userDN += "," + ldapBasedn;
//携带LDAP更多元信息以填充用户元信息
ldapAttrsInfo.put("userDN", userDN);
ldapAttrsInfo.put("sAMAccountName",
SplitUtils.keyValueSplit(si.getAttributes().get("samaccountname").toString()));
ldapAttrsInfo.put("department",
SplitUtils.keyValueSplit(si.getAttributes().get("department").toString()));
ldapAttrsInfo.put("company",
SplitUtils.keyValueSplit(si.getAttributes().get("company").toString()));
ldapAttrsInfo.put("displayName",
SplitUtils.keyValueSplit(si.getAttributes().get("displayname").toString()));
ldapAttrsInfo.put("mail",
SplitUtils.keyValueSplit(si.getAttributes().get("mail").toString()));
break;
}
}
} catch (Exception e) {
LOGGER.error("class=LdapAuthentication||method=getUserDN||account={}||errMsg={}", account, e);
}
return userDN;
return ldapAttrsInfo;
}
/**
@@ -93,23 +111,23 @@ public class LdapAuthentication {
* @param password
* @return
*/
public boolean authenticate(String account, String password) {
public Map<String, Object> authenticate(String account, String password) {
LdapContext ctx = getLdapContext();
if (ValidateUtils.isNull(ctx)) {
return false;
return null;
}
try {
String userDN = getUserDN(account, ctx);
if(ValidateUtils.isBlank(userDN)){
return false;
Map<String, Object> ldapAttrsInfo = getLdapAttrsInfo(account, ctx);
if(ValidateUtils.isNull(ldapAttrsInfo)){
return null;
}
ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN);
ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapAttrsInfo.get("userDN").toString());
ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password);
ctx.reconnect(null);
return true;
return ldapAttrsInfo;
} catch (AuthenticationException e) {
LOGGER.warn("class=LdapAuthentication||method=authenticate||account={}||errMsg={}", account, e);
} catch (NamingException e) {
@@ -125,6 +143,6 @@ public class LdapAuthentication {
}
}
}
return false;
return null;
}
}

25
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/component/sso/BaseSessionSignOn.java
View File

@@ -17,6 +17,7 @@ import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Map;
/**
* @author zengqiao
@@ -31,43 +32,53 @@ public class BaseSessionSignOn extends AbstractSingleSignOn {
private LdapAuthentication ldapAuthentication;
//是否开启ldap验证
@Value(value = "${account.ldap.enabled:}")
@Value(value = "${account.ldap.enabled:false}")
private Boolean accountLdapEnabled;
//ldap自动注册的默认角色。请注意：它通常来说都是低权限角色
@Value(value = "${account.ldap.auth-user-registration-role:}")
@Value(value = "${account.ldap.auth-user-registration-role:normal}")
private String authUserRegistrationRole;
//ldap自动注册是否开启
@Value(value = "${account.ldap.auth-user-registration:}")
private boolean authUserRegistration;
@Value(value = "${account.ldap.auth-user-registration:false}")
private Boolean authUserRegistration;
@Override
public Result<String> loginAndGetLdap(HttpServletRequest request, HttpServletResponse response, LoginDTO dto) {
if (ValidateUtils.isBlank(dto.getUsername()) || ValidateUtils.isNull(dto.getPassword())) {
return Result.buildFailure("Missing parameters");
}
Result<AccountDO> accountResult = accountService.getAccountDO(dto.getUsername());
//先创建空对象，看是在LDAP去做填充，还是直接查表填充
Result<AccountDO> accountResult;
//判断是否激活了LDAP验证, 若激活则也可使用ldap进行认证
if(!ValidateUtils.isNull(accountLdapEnabled) && accountLdapEnabled){
//去LDAP验证账密
if(!ldapAuthentication.authenticate(dto.getUsername(),dto.getPassword())){
Map<String, Object> ldapAttrsInfo = ldapAuthentication.authenticate(dto.getUsername(),dto.getPassword());;
if(ValidateUtils.isNull(ldapAttrsInfo)){
return Result.buildFrom(ResultStatus.LDAP_AUTHENTICATION_FAILED);
}
//LDAP验证通过，拿LDAP的sAMAccountName替换dto对象的值，便于第一次自动注册采用LDAP值，并且第二次也避免REPLACE
dto.setUsername(ldapAttrsInfo.get("sAMAccountName").toString());
accountResult = accountService.getAccountDO(dto.getUsername());
if((ValidateUtils.isNull(accountResult) || ValidateUtils.isNull(accountResult.getData())) && authUserRegistration){
//自动注册
AccountDO accountDO = new AccountDO();
accountDO.setUsername(dto.getUsername());
accountDO.setRole(AccountRoleEnum.getUserRoleEnum(authUserRegistrationRole).getRole());
accountDO.setPassword(dto.getPassword());
accountDO.setDisplayName(ldapAttrsInfo.getOrDefault("displayName", "").toString());
accountDO.setDepartment(ldapAttrsInfo.getOrDefault("department", "").toString());
accountDO.setMail(ldapAttrsInfo.getOrDefault("mail", "").toString());
accountService.createAccount(accountDO);
}
return Result.buildSuc(dto.getUsername());
}
//不走LDAP认证直接查表填充
accountResult = accountService.getAccountDO(dto.getUsername());
if (ValidateUtils.isNull(accountResult) || accountResult.failed()) {
return new Result<>(accountResult.getCode(), accountResult.getMessage());

3
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java
View File

@@ -275,6 +275,9 @@ public class AccountServiceImpl implements AccountService {
return enterpriseStaffService.searchEnterpriseStaffByKeyWord(prefix);
}
/**
* 定时刷新account信息到缓存中
*/
@Scheduled(cron ="0/5 * * * * ?")
public void flush() {
try {

15
kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/LoginServiceImpl.java
View File

@@ -14,6 +14,7 @@ import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.servlet.http.Cookie;
@@ -27,7 +28,13 @@ import javax.servlet.http.HttpSession;
*/
@Service("loginService")
public class LoginServiceImpl implements LoginService {
private final static Logger LOGGER = LoggerFactory.getLogger(LoginServiceImpl.class);
private static final Logger LOGGER = LoggerFactory.getLogger(LoginServiceImpl.class);
@Value(value = "${account.jump-login.gateway-api:false}")
private Boolean jumpLoginGatewayApi;
@Value(value = "${account.jump-login.third-part-api:false}")
private Boolean jumpLoginThirdPartApi;
@Autowired
private AccountService accountService;
@@ -75,8 +82,10 @@ public class LoginServiceImpl implements LoginService {
return false;
}
if (classRequestMappingValue.equals(ApiPrefix.API_V1_SSO_PREFIX)) {
// 白名单接口直接true
if (classRequestMappingValue.equals(ApiPrefix.API_V1_SSO_PREFIX) ||
(jumpLoginGatewayApi != null && jumpLoginGatewayApi && classRequestMappingValue.equals(ApiPrefix.GATEWAY_API_V1_PREFIX)) ||
(jumpLoginThirdPartApi != null && jumpLoginThirdPartApi && classRequestMappingValue.equals(ApiPrefix.API_V1_THIRD_PART_PREFIX))) {
// 登录接口 or 允许跳过且是跳过类型的接口，则直接跳过登录
return true;
}

4
kafka-manager-extends/kafka-manager-kcm/pom.xml
View File

@@ -28,7 +28,6 @@
<java_target_version>1.8</java_target_version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<file_encoding>UTF-8</file_encoding>
<spring-version>5.1.3.RELEASE</spring-version>
</properties>
<dependencies>
@@ -56,17 +55,14 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>

16
kafka-manager-extends/kafka-manager-kcm/src/main/java/com/xiaojukeji/kafka/manager/kcm/component/agent/n9e/N9e.java
View File

@@ -37,21 +37,24 @@ import java.util.Map;
public class N9e extends AbstractAgent {
private static final Logger LOGGER = LoggerFactory.getLogger(N9e.class);
@Value("${kcm.n9e.base-url}")
@Value("${kcm.n9e.base-url:}")
private String baseUrl;
@Value("${kcm.n9e.user-token}")
@Value("${kcm.n9e.user-token:12345678}")
private String userToken;
@Value("${kcm.n9e.account}")
@Value("${kcm.n9e.account:root}")
private String account;
@Value("${kcm.n9e.timeout}")
@Value("${kcm.n9e.timeout:300}")
private Integer timeout;
@Value("${kcm.n9e.script-file}")
@Value("${kcm.n9e.script-file:kcm_script.sh}")
private String scriptFile;
@Value("${kcm.n9e.logikm-url:}")
private String logiKMUrl;
private String script;
private static final String CREATE_TASK_URI = "/api/job-ce/tasks";
@@ -219,7 +222,8 @@ public class N9e extends AbstractAgent {
sb.append(creationTaskData.getKafkaPackageUrl()).append(",,");
sb.append(creationTaskData.getServerPropertiesName().replace(KafkaFileEnum.SERVER_CONFIG.getSuffix(), "")).append(",,");
sb.append(creationTaskData.getServerPropertiesMd5()).append(",,");
sb.append(creationTaskData.getServerPropertiesUrl());
sb.append(creationTaskData.getServerPropertiesUrl()).append(",,");
sb.append(this.logiKMUrl);
N9eCreationTask n9eCreationTask = new N9eCreationTask();
n9eCreationTask.setTitle(Constant.TASK_TITLE_PREFIX + "-集群ID:" + creationTaskData.getClusterId());

12
kafka-manager-extends/kafka-manager-kcm/src/main/resources/kcm_script.sh
View File

@@ -18,12 +18,13 @@ p_kafka_server_properties_name=${7} #server配置名
p_kafka_server_properties_md5=${8} #server配置MD5
p_kafka_server_properties_url=${9} #server配置文件下载地址
p_kafka_manager_url=${10} #LogiKM地址
#----------------------------------------配置信息------------------------------------------------------#
g_base_dir='/home'
g_cluster_task_dir=${g_base_dir}"/kafka_cluster_task/task_${p_task_id}" #部署升级路径
g_rollback_version=${g_cluster_task_dir}"/rollback_version" #回滚版本
g_new_kafka_package_name='' #最终的包名
g_kafka_manager_addr='' #kafka-manager地址
g_local_ip=`ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"`
g_hostname=${g_local_ip}
@@ -47,7 +48,7 @@ function dchat_alarm() {
# 检查并初始化环境
function check_and_init_env() {
if [ -z "${p_task_id}" -o -z "${p_cluster_task_type}" -o -z "${p_kafka_package_url}" -o -z "${p_cluster_id}" -o -z "${p_kafka_package_name}" -o -z "${p_kafka_package_md5}" -o -z "${p_kafka_server_properties_name}" -o -z "${p_kafka_server_properties_md5}" ]; then
if [ -z "${p_task_id}" -o -z "${p_cluster_task_type}" -o -z "${p_kafka_package_url}" -o -z "${p_cluster_id}" -o -z "${p_kafka_package_name}" -o -z "${p_kafka_package_md5}" -o -z "${p_kafka_server_properties_name}" -o -z "${p_kafka_server_properties_md5}" -o -z "${p_kafka_manager_url}" ]; then
ECHO_LOG "存在为空的参数不合法, 退出集群任务"
dchat_alarm "存在为空的参数不合法, 退出集群任务"
exit 1
@@ -72,11 +73,11 @@ function check_and_init_env() {
# 检查并等待集群所有的副本处于同步的状态
function check_and_wait_broker_stabled() {
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${g_kafka_manager_addr}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${p_kafka_manager_url}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
while [ "$under_replication_count" -ne 1 ]; do
ECHO_LOG "存在${under_replication_count}个副本未同步, sleep 10s"
sleep 10
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${g_kafka_manager_addr}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
under_replication_count=`curl -s -G -d "hostname="${g_hostname} ${p_kafka_manager_url}/api/v1/third-part/${p_cluster_id}/broker-stabled | python -m json.tool | grep true |wc -l`
done
ECHO_LOG "集群副本都已经处于同步的状态, 可以进行集群升级"
}
@@ -324,6 +325,7 @@ ECHO_LOG " p_kafka_package_name=${p_kafka_package_name}"
ECHO_LOG " p_kafka_package_md5=${p_kafka_package_md5}"
ECHO_LOG " p_kafka_server_properties_name=${p_kafka_server_properties_name}"
ECHO_LOG " p_kafka_server_properties_md5=${p_kafka_server_properties_md5}"
ECHO_LOG " p_kafka_manager_url=${p_kafka_manager_url}"
@@ -342,7 +344,7 @@ fi
ECHO_LOG "停kafka服务"
stop_kafka_server
ECHO_LOG "停5秒, 确保"
ECHO_LOG "再停5秒, 确保端口已释放"
sleep 5
if [ "${p_cluster_task_type}" == "0" ];then

3
kafka-manager-extends/kafka-manager-monitor/pom.xml
View File

@@ -25,7 +25,6 @@
<java_target_version>1.8</java_target_version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<file_encoding>UTF-8</file_encoding>
<spring-version>5.1.3.RELEASE</spring-version>
</properties>
<dependencies>
@@ -63,12 +62,10 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-version}</version>
</dependency>
<!-- testng -->

10
kafka-manager-extends/kafka-manager-monitor/src/main/java/com/xiaojukeji/kafka/manager/monitor/component/n9e/N9eService.java
View File

@@ -27,19 +27,19 @@ import java.util.concurrent.TimeUnit;
public class N9eService extends AbstractMonitorService {
private static final Logger LOGGER = LoggerFactory.getLogger(N9eService.class);
@Value("${monitor.n9e.nid}")
@Value("${monitor.n9e.nid:}")
private Integer monitorN9eNid;
@Value("${monitor.n9e.user-token}")
@Value("${monitor.n9e.user-token:}")
private String monitorN9eUserToken;
@Value("${monitor.n9e.mon.base-url}")
@Value("${monitor.n9e.mon.base-url:}")
private String monitorN9eMonBaseUrl;
@Value("${monitor.n9e.sink.base-url}")
@Value("${monitor.n9e.sink.base-url:}")
private String monitorN9eSinkBaseUrl;
@Value("${monitor.n9e.rdb.base-url}")
@Value("${monitor.n9e.rdb.base-url:}")
private String monitorN9eRdbBaseUrl;
private static final Cache<String, NotifyGroup> NOTIFY_GROUP_CACHE = Caffeine.newBuilder()

2
kafka-manager-extends/kafka-manager-notify/pom.xml
View File

@@ -25,7 +25,6 @@
<java_target_version>1.8</java_target_version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<file_encoding>UTF-8</file_encoding>
<spring-version>5.1.3.RELEASE</spring-version>
</properties>
<dependencies>
@@ -48,7 +47,6 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-version}</version>
</dependency>
</dependencies>
</project>

2
kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/OrderPassedNotifyService.java
View File

@@ -19,7 +19,7 @@ public class OrderPassedNotifyService implements ApplicationListener<OrderPassed
@Autowired
private AbstractNotifyService notifyService;
@Value("${notify.order.detail-url}")
@Value("${notify.order.detail-url:}")
private String orderDetailUrl;
@Async

2
kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/OrderRefusedNotifyService.java
View File

@@ -19,7 +19,7 @@ public class OrderRefusedNotifyService implements ApplicationListener<OrderRefus
@Autowired
private AbstractNotifyService notifyService;
@Value("${notify.order.detail-url}")
@Value("${notify.order.detail-url:}")
private String orderDetailUrl;
@Async

2
kafka-manager-extends/kafka-manager-openapi/pom.xml
View File

@@ -24,7 +24,6 @@
<java_target_version>1.8</java_target_version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<file_encoding>UTF-8</file_encoding>
<spring-version>5.1.3.RELEASE</spring-version>
</properties>
<description>
@@ -46,7 +45,6 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-version}</version>
</dependency>
<!-- testng -->

5
kafka-manager-extends/kafka-manager-openapi/src/main/java/com/xiaojukeji/kafka/manager/openapi/impl/ThirdPartServiceImpl.java
View File

@@ -42,6 +42,9 @@ public class ThirdPartServiceImpl implements ThirdPartService {
@Autowired
private ConsumerService consumerService;
@Autowired
private KafkaClientPool kafkaClientPool;
@Override
public Result<ConsumeHealthEnum> checkConsumeHealth(Long clusterId,
String topicName,
@@ -109,7 +112,7 @@ public class ThirdPartServiceImpl implements ThirdPartService {
Long timestamp) {
KafkaConsumer kafkaConsumer = null;
try {
kafkaConsumer = KafkaClientPool.borrowKafkaConsumerClient(clusterDO);
kafkaConsumer = kafkaClientPool.borrowKafkaConsumerClient(clusterDO);
if (ValidateUtils.isNull(kafkaConsumer)) {
return null;
}