8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2026-01-04 11:52:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

version 2.3.0

Browse Source
This commit is contained in:
zengqiao
2021-02-09 11:20:56 +08:00
parent 2672502c07
commit 5efd424172
56 changed files with 1018 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java
View File

@@ -22,6 +22,4 @@ public interface TopicDao {
List<TopicDO> listAll();
TopicDO getTopic(Long clusterId, String topicName, String appId);
TopicDO removeTopicInCache(Long clusterId, String topicName);
}

4
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AppDao.java
View File

@@ -16,8 +16,6 @@ public interface AppDao {
*/
int insert(AppDO appDO);
int insertIgnoreGatewayDB(AppDO appDO);
/**
* 删除appId
* @param appName App名称
@@ -60,6 +58,4 @@ public interface AppDao {
* @return int
*/
int updateById(AppDO appDO);
List<AppDO> listNewAll();
}

4
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java
View File

@@ -15,8 +15,6 @@ public interface AuthorityDao {
*/
int insert(AuthorityDO authorityDO);
int replaceIgnoreGatewayDB(AuthorityDO authorityDO);
/**
* 获取权限
* @param clusterId 集群id
@@ -38,7 +36,5 @@ public interface AuthorityDao {
Map<String, Map<Long, Map<String, AuthorityDO>>> getAllAuthority();
void removeAuthorityInCache(Long clusterId, String topicName);
int deleteAuthorityByTopic(Long clusterId, String topicName);
}

26
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AppDaoImpl.java
View File

@@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.dao.gateway.impl;
import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO;
import com.xiaojukeji.kafka.manager.dao.gateway.AppDao;
import com.xiaojukeji.kafka.manager.task.Constant;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
@@ -21,7 +22,7 @@ public class AppDaoImpl implements AppDao {
/**
* APP最近的一次更新时间, 更新之后的缓存
*/
private static Long APP_CACHE_LATEST_UPDATE_TIME = 0L;
private static volatile long APP_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP;
private static final Map<String, AppDO> APP_MAP = new ConcurrentHashMap<>();
@Override
@@ -29,11 +30,6 @@ public class AppDaoImpl implements AppDao {
return sqlSession.insert("AppDao.insert", appDO);
}
@Override
public int insertIgnoreGatewayDB(AppDO appDO) {
return sqlSession.insert("AppDao.insert", appDO);
}
@Override
public int deleteByName(String appName) {
return sqlSession.delete("AppDao.deleteByName", appName);
@@ -66,7 +62,12 @@ public class AppDaoImpl implements AppDao {
}
private void updateTopicCache() {
Long timestamp = System.currentTimeMillis();
long timestamp = System.currentTimeMillis();
if (timestamp + 1000 <= APP_CACHE_LATEST_UPDATE_TIME) {
// 近一秒内的请求不走db
return;
}
Date afterTime = new Date(APP_CACHE_LATEST_UPDATE_TIME);
List<AppDO> doList = sqlSession.selectList("AppDao.listAfterTime", afterTime);
@@ -76,19 +77,22 @@ public class AppDaoImpl implements AppDao {
/**
* 更新APP缓存
*/
synchronized private void updateTopicCache(List<AppDO> doList, Long timestamp) {
private synchronized void updateTopicCache(List<AppDO> doList, long timestamp) {
if (doList == null || doList.isEmpty() || APP_CACHE_LATEST_UPDATE_TIME >= timestamp) {
// 本次无数据更新, 或者本次更新过时 时, 忽略本次更新
return;
}
if (APP_CACHE_LATEST_UPDATE_TIME == Constant.START_TIMESTAMP) {
APP_MAP.clear();
}
for (AppDO elem: doList) {
APP_MAP.put(elem.getAppId(), elem);
}
APP_CACHE_LATEST_UPDATE_TIME = timestamp;
}
@Override
public List<AppDO> listNewAll() {
return sqlSession.selectList("AppDao.listNewAll");
public static void resetCache() {
APP_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP;
}
}

46
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java
View File

@@ -1,8 +1,8 @@
package com.xiaojukeji.kafka.manager.dao.gateway.impl;
import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao;
import com.xiaojukeji.kafka.manager.task.Constant;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
@@ -23,7 +23,8 @@ public class AuthorityDaoImpl implements AuthorityDao {
* Authority最近的一次更新时间, 更新之后的缓存
* <AppID, <clusterId, <TopicName, AuthorityDO>>>
*/
private static Long AUTHORITY_CACHE_LATEST_UPDATE_TIME = 0L;
private static volatile long AUTHORITY_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP;
private static final Map<String, Map<Long, Map<String, AuthorityDO>>> AUTHORITY_MAP = new ConcurrentHashMap<>();
@Override
@@ -31,11 +32,6 @@ public class AuthorityDaoImpl implements AuthorityDao {
return sqlSession.insert("AuthorityDao.replace", authorityDO);
}
@Override
public int replaceIgnoreGatewayDB(AuthorityDO authorityDO) {
return sqlSession.insert("AuthorityDao.replace", authorityDO);
}
@Override
public List<AuthorityDO> getAuthority(Long clusterId, String topicName, String appId) {
Map<String, Object> params = new HashMap<>(3);
@@ -62,8 +58,8 @@ public class AuthorityDaoImpl implements AuthorityDao {
}
List<AuthorityDO> authorityDOList = new ArrayList<>();
for (Long clusterId: doMap.keySet()) {
authorityDOList.addAll(doMap.get(clusterId).values());
for (Map.Entry<Long, Map<String, AuthorityDO>> entry: doMap.entrySet()) {
authorityDOList.addAll(entry.getValue().values());
}
return authorityDOList;
}
@@ -87,23 +83,6 @@ public class AuthorityDaoImpl implements AuthorityDao {
return AUTHORITY_MAP;
}
@Override
public void removeAuthorityInCache(Long clusterId, String topicName) {
AUTHORITY_MAP.forEach((appId, map) -> {
map.forEach((id, subMap) -> {
if (id.equals(clusterId)) {
subMap.remove(topicName);
if (subMap.isEmpty()) {
map.remove(id);
}
}
});
if (map.isEmpty()) {
AUTHORITY_MAP.remove(appId);
}
});
}
@Override
public int deleteAuthorityByTopic(Long clusterId, String topicName) {
Map<String, Object> params = new HashMap<>(2);
@@ -116,6 +95,11 @@ public class AuthorityDaoImpl implements AuthorityDao {
private void updateAuthorityCache() {
Long timestamp = System.currentTimeMillis();
if (timestamp + 1000 <= AUTHORITY_CACHE_LATEST_UPDATE_TIME) {
// 近一秒内的请求不走db
return;
}
Date afterTime = new Date(AUTHORITY_CACHE_LATEST_UPDATE_TIME);
List<AuthorityDO> doList = sqlSession.selectList("AuthorityDao.listAfterTime", afterTime);
updateAuthorityCache(doList, timestamp);
@@ -124,11 +108,15 @@ public class AuthorityDaoImpl implements AuthorityDao {
/**
* 更新Topic缓存
*/
synchronized private void updateAuthorityCache(List<AuthorityDO> doList, Long timestamp) {
private synchronized void updateAuthorityCache(List<AuthorityDO> doList, Long timestamp) {
if (doList == null || doList.isEmpty() || AUTHORITY_CACHE_LATEST_UPDATE_TIME >= timestamp) {
// 本次无数据更新, 或者本次更新过时 时, 忽略本次更新
return;
}
if (AUTHORITY_CACHE_LATEST_UPDATE_TIME == Constant.START_TIMESTAMP) {
AUTHORITY_MAP.clear();
}
for (AuthorityDO elem: doList) {
Map<Long, Map<String, AuthorityDO>> doMap =
AUTHORITY_MAP.getOrDefault(elem.getAppId(), new ConcurrentHashMap<>());
@@ -139,4 +127,8 @@ public class AuthorityDaoImpl implements AuthorityDao {
}
AUTHORITY_CACHE_LATEST_UPDATE_TIME = timestamp;
}
public static void resetCache() {
AUTHORITY_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP;
}
}

30
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java
View File

@@ -2,6 +2,7 @@ package com.xiaojukeji.kafka.manager.dao.impl;
import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO;
import com.xiaojukeji.kafka.manager.dao.TopicDao;
import com.xiaojukeji.kafka.manager.task.Constant;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
@@ -18,7 +19,8 @@ public class TopicDaoImpl implements TopicDao {
/**
* Topic最近的一次更新时间, 更新之后的缓存
*/
private static Long TOPIC_CACHE_LATEST_UPDATE_TIME = 0L;
private static volatile long TOPIC_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP;
private static final Map<Long, Map<String, TopicDO>> TOPIC_MAP = new ConcurrentHashMap<>();
@Autowired
@@ -62,7 +64,7 @@ public class TopicDaoImpl implements TopicDao {
@Override
public List<TopicDO> getByClusterId(Long clusterId) {
updateTopicCache();
return new ArrayList<>(TOPIC_MAP.getOrDefault(clusterId, new ConcurrentHashMap<>(0)).values());
return new ArrayList<>(TOPIC_MAP.getOrDefault(clusterId, Collections.emptyMap()).values());
}
@Override
@@ -75,28 +77,28 @@ public class TopicDaoImpl implements TopicDao {
updateTopicCache();
List<TopicDO> doList = new ArrayList<>();
for (Long clusterId: TOPIC_MAP.keySet()) {
doList.addAll(TOPIC_MAP.getOrDefault(clusterId, new ConcurrentHashMap<>(0)).values());
doList.addAll(TOPIC_MAP.getOrDefault(clusterId, Collections.emptyMap()).values());
}
return doList;
}
@Override
public TopicDO getTopic(Long clusterId, String topicName, String appId) {
Map<String, Object> params = new HashMap<>(2);
Map<String, Object> params = new HashMap<>(3);
params.put("clusterId", clusterId);
params.put("topicName", topicName);
params.put("appId", appId);
return sqlSession.selectOne("TopicDao.getTopic", params);
}
@Override
public TopicDO removeTopicInCache(Long clusterId, String topicName) {
return TOPIC_MAP.getOrDefault(clusterId, new HashMap<>(0)).remove(topicName);
}
private void updateTopicCache() {
Long timestamp = System.currentTimeMillis();
if (timestamp + 1000 <= TOPIC_CACHE_LATEST_UPDATE_TIME) {
// 近一秒内的请求不走db
return;
}
Date afterTime = new Date(TOPIC_CACHE_LATEST_UPDATE_TIME);
List<TopicDO> doList = sqlSession.selectList("TopicDao.listAfterTime", afterTime);
updateTopicCache(doList, timestamp);
@@ -105,11 +107,15 @@ public class TopicDaoImpl implements TopicDao {
/**
* 更新Topic缓存
*/
synchronized private void updateTopicCache(List<TopicDO> doList, Long timestamp) {
private synchronized void updateTopicCache(List<TopicDO> doList, Long timestamp) {
if (doList == null || doList.isEmpty() || TOPIC_CACHE_LATEST_UPDATE_TIME >= timestamp) {
// 本次无数据更新, 或者本次更新过时 时, 忽略本次更新
return;
}
if (TOPIC_CACHE_LATEST_UPDATE_TIME == Constant.START_TIMESTAMP) {
TOPIC_MAP.clear();
}
for (TopicDO elem: doList) {
Map<String, TopicDO> doMap = TOPIC_MAP.getOrDefault(elem.getClusterId(), new ConcurrentHashMap<>());
doMap.put(elem.getTopicName(), elem);
@@ -117,4 +123,8 @@ public class TopicDaoImpl implements TopicDao {
}
TOPIC_CACHE_LATEST_UPDATE_TIME = timestamp;
}
public static void resetCache() {
TOPIC_CACHE_LATEST_UPDATE_TIME = Constant.START_TIMESTAMP;
}
}

5
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/Constant.java Normal file
View File

@@ -0,0 +1,5 @@
package com.xiaojukeji.kafka.manager.task;
public class Constant {
public static final long START_TIMESTAMP = 0;
}

41
kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/task/DaoBackgroundTask.java Normal file
View File

@@ -0,0 +1,41 @@
package com.xiaojukeji.kafka.manager.task;
import com.xiaojukeji.kafka.manager.common.utils.factory.DefaultThreadFactory;
import com.xiaojukeji.kafka.manager.dao.gateway.impl.AppDaoImpl;
import com.xiaojukeji.kafka.manager.dao.gateway.impl.AuthorityDaoImpl;
import com.xiaojukeji.kafka.manager.dao.impl.TopicDaoImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* 后台任务线程
* @author zengqiao
* @date 21/02/02
*/
@Service
public class DaoBackgroundTask {
private static final Logger LOGGER = LoggerFactory.getLogger(DaoBackgroundTask.class);
private static final ScheduledExecutorService SYNC_CACHE_THREAD_POOL = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("syncCacheTask"));
@PostConstruct
public void init() {
SYNC_CACHE_THREAD_POOL.scheduleAtFixedRate(() -> {
LOGGER.info("class=DaoBackgroundTask||method=init||msg=sync cache start");
TopicDaoImpl.resetCache();
AppDaoImpl.resetCache();
AuthorityDaoImpl.resetCache();
LOGGER.info("class=DaoBackgroundTask||method=init||msg=sync cache finished");
}, 1, 10, TimeUnit.MINUTES);
}
}

8
kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml
View File

@@ -8,6 +8,7 @@
<result column="name" property="name" />
<result column="value" property="value" />
<result column="version" property="version" />
<result column="description" property="description" />
<result column="create_time" property="createTime" />
<result column="modify_time" property="modifyTime" />
</resultMap>
@@ -27,9 +28,9 @@
<insert id="insert" parameterType="com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO">
<![CDATA[
INSERT INTO gateway_config
(`type`, name, value, version)
(`type`, name, value, version, description)
VALUES
(#{type}, #{name}, #{value}, #{version})
(#{type}, #{name}, #{value}, #{version}, #{description})
]]>
</insert>
@@ -45,7 +46,8 @@
`type`=#{type},
`name`=#{name},
`value`=#{value},
`version`=#{version}
`version`=#{version},
`description`=#{description}
WHERE id=#{id}
]]>
</update>