From e1514c901bd53d2a3eb07b2346bff1fc20574d72 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 23 Aug 2022 19:01:53 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/topic/impl/TopicStateManagerImpl.java | 2 +- .../metric/AbstractMetricCollector.java | 28 --- .../metric/BrokerMetricCollector.java | 2 - .../metric/ClusterMetricCollector.java | 1 - .../metric/GroupMetricCollector.java | 1 - .../km/collector/metric/MetricESSender.java | 44 ++--- .../metric/PartitionMetricCollector.java | 2 - .../metric/ReplicaMetricCollector.java | 2 - .../metric/TopicMetricCollector.java | 2 - .../bean/entity/metrics/PartitionMetrics.java | 2 +- km-console/.env | 2 + km-console/package.json | 6 +- .../config/d1-webpack.base.js | 27 +-- .../packages/config-manager-fe/package.json | 3 +- .../src/constants/axiosConfig.ts | 9 +- .../src/pages/CommonConfig.tsx | 22 ++- .../src/pages/ConfigManage/index.tsx | 6 +- .../src/pages/OperationLog/index.tsx | 2 +- .../src/pages/UserManage/UserTabContent.tsx | 16 +- .../config-manager-fe/webpack.config.js | 22 +-- .../config/CoverHtmlWebpackPlugin.js | 51 ++++-- .../config/d1-spa-webpack.js | 162 ------------------ .../config/d1-webpack.base.js | 86 +++++----- .../layout-clusters-fe/config/registerApps.js | 1 + .../config/systemsConfig.js | 8 +- .../packages/layout-clusters-fe/favicon.ico | Bin 1315 -> 4286 bytes .../packages/layout-clusters-fe/package.json | 3 +- .../components/CodeMirrorFormItem/index.less | 7 + .../DashboardDragChart/ChartDetail.tsx | 8 +- .../components/DashboardDragChart/config.tsx | 5 +- .../components/DashboardDragChart/index.less | 93 +++++----- .../SingleChartHeader/IndicatorDrawer.tsx | 2 + .../style/indicator-drawer.less | 6 + .../components/TopicJob/PreviewTaskPlan.tsx | 4 +- .../src/components/TopicJob/ReplicaChange.tsx | 8 +- .../src/components/TopicJob/ReplicaMove.tsx | 9 +- .../src/components/TourGuide/index.less | 17 ++ .../src/components/TourGuide/index.tsx | 24 ++- .../src/components/TourGuide/steps.ts | 3 + .../src/constants/axiosConfig.ts | 3 +- .../src/constants/chartConfig.ts | 34 +++- .../layout-clusters-fe/src/locales/zh.tsx | 2 +- .../pages/Consumers/ConsumerGroupDetail.tsx | 57 ++---- .../src/pages/Consumers/index.tsx | 2 +- .../src/pages/Jobs/RebalancePlan.tsx | 28 ++- .../src/pages/Jobs/index.less | 6 +- .../src/pages/Jobs/index.tsx | 10 +- .../src/pages/MutliClusterPage/index.less | 2 +- .../pages/SingleClusterDetail/ChangeLog.tsx | 6 +- .../src/pages/SingleClusterDetail/config.tsx | 22 +-- .../src/pages/TestingProduce/Produce.tsx | 1 + .../TestingProduce/component/style/form.less | 8 + .../src/pages/TestingProduce/config.tsx | 2 + .../src/pages/TopicDetail/Configuration.tsx | 2 +- .../src/pages/TopicDetail/config.tsx | 32 ++-- .../src/pages/TopicDetail/index.less | 4 +- .../src/pages/TopicList/Delete.tsx | 8 + .../src/pages/TopicList/index.less | 1 + .../src/pages/TopicList/index.tsx | 4 +- .../src/style-addition.less | 6 +- .../static/js/echarts.min.js | 45 +++++ .../static/js/history.production.min.js | 2 + .../static/js/lodash.min.js | 140 +++++++++++++++ .../static/js/react-router-dom.min.js | 2 + .../static/js/react-router.min.js | 2 + .../layout-clusters-fe/static/logo-white.png | Bin 10165 -> 0 bytes .../layout-clusters-fe/webpack.config.js | 62 +++---- ...e.java => CollectedMetricsLocalCache.java} | 18 +- .../broker/impl/BrokerMetricServiceImpl.java | 6 +- .../partition/PartitionMetricService.java | 1 + .../impl/PartitionMetricServiceImpl.java | 37 +++- .../reassign/impl/ReassignJobServiceImpl.java | 98 ++++++++++- .../impl/ReplicaMetricServiceImpl.java | 6 +- .../topic/impl/TopicMetricServiceImpl.java | 6 +- .../templates/configMap.yaml | 4 +- .../km/task/job/CommunityReassignJobTask.java | 6 + 76 files changed, 796 insertions(+), 577 deletions(-) create mode 100644 km-console/.env delete mode 100644 km-console/packages/layout-clusters-fe/config/d1-spa-webpack.js create mode 100644 km-console/packages/layout-clusters-fe/static/js/echarts.min.js create mode 100644 km-console/packages/layout-clusters-fe/static/js/history.production.min.js create mode 100644 km-console/packages/layout-clusters-fe/static/js/lodash.min.js create mode 100644 km-console/packages/layout-clusters-fe/static/js/react-router-dom.min.js create mode 100644 km-console/packages/layout-clusters-fe/static/js/react-router.min.js delete mode 100755 km-console/packages/layout-clusters-fe/static/logo-white.png rename km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/cache/{CollectMetricsLocalCache.java => CollectedMetricsLocalCache.java} (73%) diff --git a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/impl/TopicStateManagerImpl.java b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/impl/TopicStateManagerImpl.java index 0e0902f2..4e892f94 100644 --- a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/impl/TopicStateManagerImpl.java +++ b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/impl/TopicStateManagerImpl.java @@ -241,7 +241,7 @@ public class TopicStateManagerImpl implements TopicStateManager { return Result.buildSuc(); } - Result> metricsResult = partitionMetricService.getLatestMetricsFromES(clusterPhyId, topicName, metricsNames); + Result> metricsResult = partitionMetricService.collectPartitionsMetricsFromKafka(clusterPhyId, topicName, metricsNames); if (metricsResult.failed()) { // 仅打印错误日志,但是不直接返回错误 log.error( diff --git a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/AbstractMetricCollector.java b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/AbstractMetricCollector.java index b9f25f66..7d0e85f6 100644 --- a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/AbstractMetricCollector.java +++ b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/AbstractMetricCollector.java @@ -1,28 +1,18 @@ package com.xiaojukeji.know.streaming.km.collector.metric; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; import com.xiaojukeji.know.streaming.km.collector.service.CollectThreadPoolService; import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; -import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics; import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BaseMetricEvent; import com.xiaojukeji.know.streaming.km.common.component.SpringTool; import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum; import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; import org.springframework.beans.factory.annotation.Autowired; -import java.util.concurrent.TimeUnit; /** * @author didi */ public abstract class AbstractMetricCollector { - private static final double SIZE_THRESHOLD = 0.8; - - private final Cache latestMetricsMap = Caffeine.newBuilder() - .expireAfterWrite(3, TimeUnit.MINUTES) - .build(); - public abstract void collectMetrics(ClusterPhy clusterPhy); public abstract VersionItemTypeEnum collectorType(); @@ -30,24 +20,6 @@ public abstract class AbstractMetricCollector { @Autowired private CollectThreadPoolService collectThreadPoolService; - /** - * 如果最近3分钟内的指标有异常,则采用之前的值 - */ - protected void doOptimizeMetric(BaseMetrics metricPO){ - BaseMetrics latestMetrics = latestMetricsMap.getIfPresent(metricPO.unique()); - if (latestMetrics == null) { - latestMetrics = metricPO; - } - - if(metricPO.getMetrics().size() < latestMetrics.getMetrics().size() * SIZE_THRESHOLD) { - // 异常采集时,则替换metrics - metricPO.putMetric(latestMetrics.getMetrics()); - } else { - // 正常采集时,则替换cache - latestMetricsMap.put(metricPO.unique(), metricPO); - } - } - protected FutureWaitUtil getFutureUtilByClusterPhyId(Long clusterPhyId) { return collectThreadPoolService.selectSuitableFutureUtil(clusterPhyId * 1000L + this.collectorType().getCode()); } diff --git a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/BrokerMetricCollector.java b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/BrokerMetricCollector.java index 16fc7955..e60372a4 100644 --- a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/BrokerMetricCollector.java +++ b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/BrokerMetricCollector.java @@ -103,8 +103,6 @@ public class BrokerMetricCollector extends AbstractMetricCollector { @Override public void onApplicationEvent(BaseMetricEvent event) { - if(event instanceof BrokerMetricEvent){ + if(event instanceof BrokerMetricEvent) { BrokerMetricEvent brokerMetricEvent = (BrokerMetricEvent)event; send2es(KafkaMetricIndexEnum.BROKER_INFO, - ConvertUtil.list2List(brokerMetricEvent.getBrokerMetrics(), BrokerMetricPO.class)); + ConvertUtil.list2List(brokerMetricEvent.getBrokerMetrics(), BrokerMetricPO.class) + ); - }else if(event instanceof ClusterMetricEvent){ + } else if(event instanceof ClusterMetricEvent) { ClusterMetricEvent clusterMetricEvent = (ClusterMetricEvent)event; send2es(KafkaMetricIndexEnum.CLUSTER_INFO, - ConvertUtil.list2List(clusterMetricEvent.getClusterMetrics(), ClusterMetricPO.class)); + ConvertUtil.list2List(clusterMetricEvent.getClusterMetrics(), ClusterMetricPO.class) + ); - }else if(event instanceof TopicMetricEvent){ + } else if(event instanceof TopicMetricEvent) { TopicMetricEvent topicMetricEvent = (TopicMetricEvent)event; send2es(KafkaMetricIndexEnum.TOPIC_INFO, - ConvertUtil.list2List(topicMetricEvent.getTopicMetrics(), TopicMetricPO.class)); + ConvertUtil.list2List(topicMetricEvent.getTopicMetrics(), TopicMetricPO.class) + ); - }else if(event instanceof PartitionMetricEvent){ + } else if(event instanceof PartitionMetricEvent) { PartitionMetricEvent partitionMetricEvent = (PartitionMetricEvent)event; send2es(KafkaMetricIndexEnum.PARTITION_INFO, - ConvertUtil.list2List(partitionMetricEvent.getPartitionMetrics(), PartitionMetricPO.class)); + ConvertUtil.list2List(partitionMetricEvent.getPartitionMetrics(), PartitionMetricPO.class) + ); - }else if(event instanceof GroupMetricEvent){ + } else if(event instanceof GroupMetricEvent) { GroupMetricEvent groupMetricEvent = (GroupMetricEvent)event; send2es(KafkaMetricIndexEnum.GROUP_INFO, - ConvertUtil.list2List(groupMetricEvent.getGroupMetrics(), GroupMetricPO.class)); + ConvertUtil.list2List(groupMetricEvent.getGroupMetrics(), GroupMetricPO.class) + ); - }else if(event instanceof ReplicaMetricEvent){ + } else if(event instanceof ReplicaMetricEvent) { ReplicaMetricEvent replicaMetricEvent = (ReplicaMetricEvent)event; send2es(KafkaMetricIndexEnum.REPLICATION_INFO, - ConvertUtil.list2List(replicaMetricEvent.getReplicationMetrics(), ReplicationMetricPO.class)); + ConvertUtil.list2List(replicaMetricEvent.getReplicationMetrics(), ReplicationMetricPO.class) + ); } } /** * 根据不同监控维度来发送 - * - * @param stats - * @param statsList - * @return */ private boolean send2es(KafkaMetricIndexEnum stats, List statsList){ if (CollectionUtils.isEmpty(statsList)) { @@ -98,8 +100,9 @@ public class MetricESSender implements ApplicationListener { int num = (size) % THRESHOLD == 0 ? (size / THRESHOLD) : (size / THRESHOLD + 1); if (size < THRESHOLD) { - esExecutor.execute(() -> - baseMetricESDao.batchInsertStats(statsList)); + esExecutor.execute( + () -> baseMetricESDao.batchInsertStats(statsList) + ); return true; } @@ -107,8 +110,9 @@ public class MetricESSender implements ApplicationListener { int end = (i * THRESHOLD) > size ? size : (i * THRESHOLD); int start = (i - 1) * THRESHOLD; - esExecutor.execute(() -> - baseMetricESDao.batchInsertStats(statsList.subList(start, end))); + esExecutor.execute( + () -> baseMetricESDao.batchInsertStats(statsList.subList(start, end)) + ); } return true; diff --git a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/PartitionMetricCollector.java b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/PartitionMetricCollector.java index bcd451a2..89363652 100644 --- a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/PartitionMetricCollector.java +++ b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/PartitionMetricCollector.java @@ -124,7 +124,5 @@ public class PartitionMetricCollector extends AbstractMetricCollector doOptimizeMetric(elem.getValue())); } } diff --git a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/ReplicaMetricCollector.java b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/ReplicaMetricCollector.java index 13874121..5f712f93 100644 --- a/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/ReplicaMetricCollector.java +++ b/km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/ReplicaMetricCollector.java @@ -116,8 +116,6 @@ public class ReplicaMetricCollector extends AbstractMetricCollector { const manifestName = `manifest.json`; - const jsFileName = isProd ? '[name]-[chunkhash].js' : '[name].js'; const cssFileName = isProd ? '[name]-[chunkhash].css' : '[name].css'; const plugins = [ - // !isProd && new HardSourceWebpackPlugin(), new ProgressBarPlugin(), new CaseSensitivePathsPlugin(), new MiniCssExtractPlugin({ @@ -83,18 +78,28 @@ module.exports = () => { new ReactRefreshWebpackPlugin({ overlay: false, }), + // new BundleAnalyzerPlugin({ + // analyzerPort: 8889 + // }), ].filter(Boolean); if (isProd) { plugins.push(new CleanWebpackPlugin()); } return { - output: { - filename: jsFileName, - chunkFilename: jsFileName, - publicPath, - }, externals: isProd - ? [/^react$/, /^react\/lib.*/, /^react-dom$/, /.*react-dom.*/, /^single-spa$/, /^single-spa-react$/, /^moment$/, /^antd$/, /^lodash$/] + ? [ + /^react$/, + /^react\/lib.*/, + /^react-dom$/, + /.*react-dom.*/, + /^single-spa$/, + /^single-spa-react$/, + /^moment$/, + /^antd$/, + /^lodash$/, + /^react-router$/, + /^react-router-dom$/, + ] : [], resolve: { symlinks: false, diff --git a/km-console/packages/config-manager-fe/package.json b/km-console/packages/config-manager-fe/package.json index b04ec4a1..00d8c3b8 100644 --- a/km-console/packages/config-manager-fe/package.json +++ b/km-console/packages/config-manager-fe/package.json @@ -23,6 +23,7 @@ "dependencies": { "babel-preset-react-app": "^10.0.0", "classnames": "^2.2.6", + "dotenv": "^16.0.1", "less": "^3.9.0", "lodash": "^4.17.11", "mobx": "4.15.7", @@ -99,4 +100,4 @@ "webpack-dev-server": "^3.2.1", "webpack-merge": "^4.2.1" } -} \ No newline at end of file +} diff --git a/km-console/packages/config-manager-fe/src/constants/axiosConfig.ts b/km-console/packages/config-manager-fe/src/constants/axiosConfig.ts index 24b4d2b0..f1b04eae 100644 --- a/km-console/packages/config-manager-fe/src/constants/axiosConfig.ts +++ b/km-console/packages/config-manager-fe/src/constants/axiosConfig.ts @@ -3,9 +3,10 @@ import { notification, Utils } from 'knowdesign'; -const goLogin = () => { - // notification.error({ message: '当前未登录,将自动跳转到登录页' }); - window.history.replaceState({}, '', `/login?redirect=${window.location.href.slice(window.location.origin.length)}`); +export const goLogin = () => { + if (!window.location.pathname.toLowerCase().startsWith('/login')) { + window.history.replaceState({}, '', `/login?redirect=${window.location.href.slice(window.location.origin.length)}`); + } }; const serviceInstance = Utils.service; @@ -18,7 +19,7 @@ serviceInstance.interceptors.request.use( (config: any) => { const user = Utils.getCookie('X-SSO-USER'); const id = Utils.getCookie('X-SSO-USER-ID'); - if ((!user || !id) && !window.location.pathname.toLowerCase().startsWith('/login')) { + if (!user || !id) { goLogin(); } else { config.headers['X-SSO-USER'] = user; // 请求携带token diff --git a/km-console/packages/config-manager-fe/src/pages/CommonConfig.tsx b/km-console/packages/config-manager-fe/src/pages/CommonConfig.tsx index 8f48bc8e..3d5b0a01 100644 --- a/km-console/packages/config-manager-fe/src/pages/CommonConfig.tsx +++ b/km-console/packages/config-manager-fe/src/pages/CommonConfig.tsx @@ -1,5 +1,6 @@ import React, { useLayoutEffect } from 'react'; import { Utils, AppContainer } from 'knowdesign'; +import { goLogin } from 'constants/axiosConfig'; // 权限对应表 export enum ConfigPermissionMap { @@ -33,12 +34,7 @@ const CommonConfig = (): JSX.Element => { const [global, setGlobal] = AppContainer.useGlobalValue(); // 获取权限树 - const getPermissionTree = () => { - // 如果未登录,直接退出 - const userInfo = localStorage.getItem('userInfo'); - if (!userInfo) return false; - - const userId = JSON.parse(userInfo).id; + const getPermissionTree = (userId: number) => { const getUserInfo = Utils.request(`/logi-security/api/v1/user/${userId}`); const getPermissionTree = Utils.request('/logi-security/api/v1/permission/tree'); @@ -58,7 +54,19 @@ const CommonConfig = (): JSX.Element => { }; useLayoutEffect(() => { - getPermissionTree(); + // 如果未登录,直接跳转到登录页 + const userInfo = localStorage.getItem('userInfo'); + let userId: number; + + try { + userId = JSON.parse(userInfo).id; + if (!userId) throw 'err'; + } catch (_) { + goLogin(); + return; + } + + getPermissionTree(userId); }, []); return <>; diff --git a/km-console/packages/config-manager-fe/src/pages/ConfigManage/index.tsx b/km-console/packages/config-manager-fe/src/pages/ConfigManage/index.tsx index 70a02ff6..84b5d2f6 100644 --- a/km-console/packages/config-manager-fe/src/pages/ConfigManage/index.tsx +++ b/km-console/packages/config-manager-fe/src/pages/ConfigManage/index.tsx @@ -433,13 +433,13 @@ export default () => {
getConfigList({ page: 1 })}> - - + - +