合并主分支 (#1241)

1：topic维度的查询BytesIn，BytesOut 聚合类型应该是 sum 不能是 avg
2：getAggListMetrics dsl需要加上 brokerAgg = 1的条件

README.md

@@ -136,7 +136,7 @@
 
 👍 我们正在组建国内最大，最权威的 **[Kafka中文社区](https://z.didi.cn/5gSF9)**
 
-在这里你可以结交各大互联网的 Kafka大佬 以及 4000+ Kafka爱好者，一起实现知识共享，实时掌控最新行业资讯，期待 👏   您的加入中～ https://z.didi.cn/5gSF9
+在这里你可以结交各大互联网的 Kafka大佬 以及 6200+ Kafka爱好者，一起实现知识共享，实时掌控最新行业资讯，期待 👏   您的加入中～ https://z.didi.cn/5gSF9
 
 有问必答~！ 互动有礼~！
 
@@ -146,7 +146,7 @@ PS: 提问请尽量把问题一次性描述清楚，并告知环境信息情况
 
 **`2、微信群`**
 
-微信加群：添加`PenceXie` 、`szzdzhp001`的微信号备注KnowStreaming加群。
+微信加群：添加`PenceXie` 的微信号备注KnowStreaming加群。
 <br/>
 
 加群之前有劳点一下 star，一个小小的 star 是对KnowStreaming作者们努力建设社区的动力。

docs/dev_guide/MYSQL密码加密手册.md

@@ -0,0 +1,115 @@
+## YML文件MYSQL密码加密存储手册
+
+### 1、本地部署加密
+
+**第一步：生成密文**
+
+在本地仓库中找到jasypt-1.9.3.jar，默认在org/jasypt/jasypt/1.9.3中，使用`java -cp`生成密文。
+
+```bash
+java -cp jasypt-1.9.3.jar org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI input=mysql密码 password=加密的salt algorithm=PBEWithMD5AndDES
+```
+
+```bash
+## 得到密文
+DYbVDLg5D0WRcJSCUGWjiw==
+```
+
+**第二步：配置jasypt**
+
+在YML文件中配置jasypt，例如
+
+```yaml
+jasypt:
+  encryptor:
+    algorithm: PBEWithMD5AndDES
+    iv-generator-classname: org.jasypt.iv.NoIvGenerator
+```
+
+**第三步：配置密文**
+
+使用密文替换YML文件中的明文密码为ENC(密文），例如[application.yml](https://github.com/didi/KnowStreaming/blob/master/km-rest/src/main/resources/application.yml)中MYSQL密码。
+
+```yaml
+know-streaming: 
+  username: root
+  password: ENC(DYbVDLg5D0WRcJSCUGWjiw==)
+```
+
+**第四步：配置加密的salt（选择其一）**
+
+- 配置在YML文件中（不推荐）
+
+```yaml
+jasypt:
+  encryptor:
+    password: salt
+```
+
+- 配置程序启动时的命令行参数
+
+```bash
+java -jar xxx.jar --jasypt.encryptor.password=salt
+```
+
+- 配置程序启动时的环境变量
+
+```bash
+export JASYPT_PASSWORD=salt
+java -jar xxx.jar --jasypt.encryptor.password=${JASYPT_PASSWORD}
+```
+
+## 2、容器部署加密
+
+利用docker swarm 提供的 secret 机制加密存储密码，使用docker swarm来管理密码。
+
+### 2.1、secret加密存储
+
+**第一步：初始化docker swarm**
+
+```bash
+docker swarm init
+```
+
+**第二步：创建密钥**
+
+```bash
+echo "admin2022_" | docker secret create mysql_password -
+
+# 输出密钥
+f964wi4gg946hu78quxsh2ge9
+```
+
+**第三步：使用密钥**
+
+```yaml
+# mysql用户密码
+SERVER_MYSQL_USER: root
+SERVER_MYSQL_PASSWORD: mysql_password
+
+knowstreaming-mysql:
+    # root 用户密码
+    MYSQL_ROOT_PASSWORD: mysql_password
+secrets:
+  mysql_password:
+    external: true
+```
+
+### 2.2、使用密钥文件加密
+
+**第一步：创建密钥**
+
+```bash
+echo "admin2022_" > password
+```
+
+**第二步：使用密钥**
+
+```yaml
+# mysql用户密码
+SERVER_MYSQL_USER: root
+SERVER_MYSQL_PASSWORD: mysql_password
+secrets:
+  mysql_password:
+    file: ./password
+```

km-console/packages/config-manager-fe/package.json

@@ -35,7 +35,7 @@
   },
   "devDependencies": {
     "@ant-design/icons": "^4.6.2",
-    "@babel/core": "^7.18.13",
+    "@babel/core": "^7.5.5",
     "@babel/plugin-proposal-class-properties": "^7.4.0",
     "@babel/plugin-proposal-decorators": "^7.4.0",
     "@babel/plugin-proposal-export-default-from": "^7.2.0",
@@ -50,7 +50,7 @@
     "@babel/preset-react": "^7.0.0",
     "@babel/preset-typescript": "^7.14.5",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.1",
-    "@types/lodash": "^4.14.184",
+    "@types/lodash": "^4.14.138",
     "@types/react-dom": "^17.0.5",
     "@types/react-router": "5.1.18",
     "@types/react-router-dom": "^5.3.3",
@@ -76,7 +76,6 @@
     "hard-source-webpack-plugin": "^0.13.1",
     "html-webpack-plugin": "^4.0.0",
     "husky": "4.3.7",
-    "knowdesign": "^1.3.7",
     "less-loader": "^4.1.0",
     "lint-staged": "10.5.3",
     "mini-css-extract-plugin": "^1.3.0",

km-console/packages/layout-clusters-fe/config/CoverHtmlWebpackPlugin.js

@@ -109,15 +109,7 @@ class CoverHtmlWebpackPlugin {
                   <script src='${isProd ? PublicPath : ''}/static/js/named-exports.min.js'></script>
                   <script src='${isProd ? PublicPath : ''}/static/js/use-default.min.js'></script>
                   <script src='${isProd ? PublicPath : ''}/static/js/amd.js'></script>
-                  <script>
+                  ${process.env.BUSINESS_VERSION === 'true' ? `<script src=${isProd ? PublicPath : ''}/static/js/ksl.min.js></script>` : ''}
-                    var _hmt = _hmt || [];
-                    (function() {
-                      var hm = document.createElement("script");
-                      hm.src = "https://hm.baidu.com/hm.js?16d7da6d1dd79237d801ee55809cfe90";
-                      var s = document.getElementsByTagName("script")[0]; 
-                      s.parentNode.insertBefore(hm, s);
-                    })();
-                  </script>
                 </head>
                 <body>
                   ${depsMap}

km-console/packages/layout-clusters-fe/package.json

@@ -64,7 +64,7 @@
     "webpack-bundle-analyzer": "^4.5.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.13",
+    "@babel/core": "^7.5.5",
     "@babel/plugin-proposal-class-properties": "^7.4.0",
     "@babel/plugin-proposal-decorators": "^7.4.0",
     "@babel/plugin-proposal-export-default-from": "^7.2.0",
@@ -79,7 +79,7 @@
     "@babel/preset-typescript": "^7.14.5",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.1",
     "@types/crypto-js": "^4.1.0",
-    "@types/lodash": "^4.14.184",
+    "@types/lodash": "^4.14.171",
     "@types/node": "^12.12.25",
     "@types/pubsub-js": "^1.5.18",
     "pubsub-js": "^1.5.18",
@@ -101,7 +101,6 @@
     "file-loader": "^6.0.0",
     "hard-source-webpack-plugin": "^0.13.1",
     "husky": "4.3.7",
-    "knowdesign": "^1.3.7",
     "less": "^3.9.0",
     "less-loader": "^4.1.0",
     "lint-staged": "10.5.3",

km-console/packages/layout-clusters-fe/src/app.tsx

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
 /* eslint-disable no-constant-condition */
 import '@babel/polyfill';
 import React, { useState, useEffect, useLayoutEffect } from 'react';
@@ -89,17 +88,6 @@ const AppContent = (props: { setlanguage: (language: string) => void }) => {
     } else {
       setCurActiveAppName('cluster');
     }
-    // @ts-ignore
-    window._hmt = window._hmt || [];
-    // window._hmt.push([
-    //   '_setPageviewProperty',
-    //   {
-    //     page_name: 'test',
-    //     page_title: null, // 当属性值传入 null 时，其作用为删除此前设置的该 PV 属性
-    //   },
-    // ]);
-    // @ts-ignore
-    window._hmt.push(['_trackPageview', pathname]);
   }, [pathname]);
 
   // 获取版本信息

km-console/packages/layout-clusters-fe/src/constants/common.ts

@@ -154,15 +154,15 @@ export const urlParser = () => {
 
 export const getLicenseInfo = (cbk: (msg: string) => void) => {
   if (process.env.BUSINESS_VERSION) {
-    // const info = (window as any).code;
+    const info = (window as any).code;
-    // if (!info) {
+    if (!info) {
-    //   setTimeout(() => getLicenseInfo(cbk), 1000);
+      setTimeout(() => getLicenseInfo(cbk), 1000);
-    // } else {
+    } else {
-    //   const res = info() || {};
+      const res = info() || {};
-    //   if (res.code !== 0) {
+      if (res.code !== 0) {
-    //     cbk(res.msg);
+        cbk(res.msg);
-    //   }
+      }
-    // }
+    }
   }
 };
 

km-console/packages/layout-clusters-fe/src/constants/menu.tsx

@@ -150,14 +150,7 @@ export const leftMenus = (clusterId?: string, clusterRunState?: number) => ({
       children: [
         process.env.BUSINESS_VERSION
           ? {
-              name: (intl: any) => {
+              name: 'balance',
-                return (
-                  <div className="menu-item-with-pro-tag">
-                    <span>{intl.formatMessage({ id: 'menu.cluster.operation.balance' })}</span>
-                    <div className="pro-tag"></div>
-                  </div>
-                );
-              },
               path: 'balance',
               icon: '#icon-luoji',
             }
@@ -171,26 +164,19 @@ export const leftMenus = (clusterId?: string, clusterRunState?: number) => ({
     },
     process.env.BUSINESS_VERSION
       ? {
-          name: (intl: any) => {
+          name: 'produce-consume',
-            return (
-              <div className="menu-item-with-pro-tag">
-                <span>{intl.formatMessage({ id: 'menu.cluster.produce-consume' })}</span>
-                <div className="pro-tag"></div>
-              </div>
-            );
-          },
           path: 'testing',
           icon: 'icon-Message',
           permissionPoint: [ClustersPermissionMap.TEST_CONSUMER, ClustersPermissionMap.TEST_PRODUCER],
           children: [
             {
-              name: (intl: any) => <span>{intl.formatMessage({ id: 'menu.cluster.produce-consume.producer' })}</span>,
+              name: 'producer',
               path: 'producer',
               icon: 'icon-luoji',
               permissionPoint: ClustersPermissionMap.TEST_PRODUCER,
             },
             {
-              name: (intl: any) => <span>{intl.formatMessage({ id: 'menu.cluster.produce-consume.consumer' })}</span>,
+              name: 'consumer',
               path: 'consumer',
               icon: 'icon-luoji',
               permissionPoints: ClustersPermissionMap.TEST_CONSUMER,

km-console/packages/layout-clusters-fe/src/index.less

@@ -259,19 +259,6 @@ li {
   }
 }
 
-.menu-item-with-pro-tag {
-  display: flex;
-  .pro-tag {
-    width: 24px;
-    margin-left: 4px;
-    background: no-repeat center/24px 16px url('./assets/pro-tag.png');
-  }
-}
-
-.dcloud-menu-item-selected .menu-item-with-pro-tag .pro-tag {
-  width: 0;
-}
-
 .menu-item-with-beta-tag {
   display: flex;
   .beta-tag {

km-console/packages/layout-clusters-fe/src/pages/Login/config.tsx

@@ -45,7 +45,6 @@ export const FormMap = [
       prefix: <></>,
       // prefix: <UserOutlined style={{ color: 'rgba(0,0,0,.25)' }} />,
     },
-    initialValue: 'admin',
   },
   {
     key: 'pw',
@@ -65,7 +64,6 @@ export const FormMap = [
       placeholder: '请输入密码',
       // prefix: <LockOutlined style={{ color: 'rgba(0,0,0,.25)' }} />,
     },
-    initialValue: 'admin2022_',
   },
 ];
 

km-console/packages/layout-clusters-fe/src/pages/Login/login.tsx

@@ -58,14 +58,7 @@ export const LoginForm: React.FC<any> = (props) => {
         {FormMap.map((formItem) => {
           return (
             <Row key={formItem.key}>
-              <Form.Item
+              <Form.Item key={formItem.key} name={formItem.key} label={formItem.label} rules={formItem.rules} style={{ width: '100%' }}>
-                key={formItem.key}
-                name={formItem.key}
-                label={formItem.label}
-                rules={formItem.rules}
-                initialValue={formItem.initialValue}
-                style={{ width: '100%' }}
-              >
                 {renderFormItem(formItem)}
               </Form.Item>
             </Row>

km-console/packages/layout-clusters-fe/src/pages/TestingConsumer/Consume.tsx

@@ -144,6 +144,8 @@ const ConsumeClientTest = () => {
             ...configInfo,
             needFilterKeyValue: changeValue === 1 || changeValue === 2,
             needFilterSize: changeValue === 3 || changeValue === 4 || changeValue === 5,
+            needFilterKey: changeValue === 6,
+            needFilterValue: changeValue === 7,
           });
           break;
       }

km-console/packages/layout-clusters-fe/src/pages/TestingConsumer/config.tsx

@@ -16,19 +16,19 @@ export const cardList = [
 
 export const filterList = [
   {
-    label: 'none',
+    label: 'None',
     value: 0,
   },
   {
-    label: 'contains',
+    label: 'Contains',
     value: 1,
   },
   {
-    label: 'does not contains',
+    label: 'Does Not Contains',
     value: 2,
   },
   {
-    label: 'equals',
+    label: 'Equals',
     value: 3,
   },
   {
@@ -39,6 +39,14 @@ export const filterList = [
     label: 'Under Size',
     value: 5,
   },
+  {
+      label: 'Key Contains',
+      value: 6,
+  },
+  {
+      label: 'Value Contains',
+      value: 7,
+  }
 ];
 
 export const untilList = [
@@ -324,10 +332,10 @@ export const getFormConfig = (topicMetaData: any, info = {} as any, partitionLis
       key: 'filterKey',
       label: 'Key',
       type: FormItemType.input,
-      invisible: !info?.needFilterKeyValue,
+      invisible: !info?.needFilterKeyValue && !info?.needFilterKey,
       rules: [
         {
-          required: info?.needFilterKeyValue,
+          required: info?.needFilterKeyValue || info?.needFilterKey,
           message: '请输入Key',
         },
       ],
@@ -336,10 +344,10 @@ export const getFormConfig = (topicMetaData: any, info = {} as any, partitionLis
       key: 'filterValue',
       label: 'Value',
       type: FormItemType.input,
-      invisible: !info?.needFilterKeyValue,
+      invisible: !info?.needFilterKeyValue && !info?.needFilterValue,
       rules: [
         {
-          required: info?.needFilterKeyValue,
+          required: info?.needFilterKeyValue || info?.needFilterValue,
           message: '请输入Value',
         },
       ],

km-console/packages/layout-clusters-fe/src/pages/TopicList/ExpandPartition.tsx

@@ -44,7 +44,7 @@ const ExpandPartition = (props: { record: any; onConfirm: () => void }) => {
 
     setLoading(true);
     const metricParams = {
-      aggType: 'avg',
+      aggType: 'sum',
       endTime: Math.round(endStamp),
       metricsNames: ['BytesIn', 'BytesOut'],
       startTime: Math.round(startStamp),

km-console/pom.xml

@@ -32,8 +32,6 @@
                         <configuration>
                             <nodeVersion>v12.22.12</nodeVersion>
                             <npmVersion>6.14.16</npmVersion>
-                            <nodeDownloadRoot>https://npm.taobao.org/mirrors/node/</nodeDownloadRoot>
-                            <npmDownloadRoot>https://registry.npm.taobao.org/npm/-/</npmDownloadRoot>
                         </configuration>
                     </execution>
                     <execution>

km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/broker/impl/BrokerConfigServiceImpl.java

@@ -37,6 +37,7 @@ import scala.jdk.javaapi.CollectionConverters;
 
 import javax.annotation.PostConstruct;
 import java.util.*;
+import java.util.stream.Collectors;
 
 import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
 
@@ -154,9 +155,11 @@ public class BrokerConfigServiceImpl extends BaseKafkaVersionControlService impl
         if (propertiesResult.failed()) {
             return Result.buildFromIgnoreData(propertiesResult);
         }
+        List<String> configKeyList = propertiesResult.getData().keySet().stream().map(Object::toString).collect(Collectors.toList());
+
 
         return Result.buildSuc(KafkaConfigConverter.convert2KafkaBrokerConfigDetailList(
-                new ArrayList<>(),
+                configKeyList,
                 propertiesResult.getData()
         ));
     }

km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/biz/impl/KafkaClientTestManagerImpl.java

@@ -451,6 +451,18 @@ public class KafkaClientTestManagerImpl implements KafkaClientTestManager {
             return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "包含的方式过滤，必须有过滤的key或value");
         }
 
+        // key包含过滤
+        if (KafkaConsumerFilterEnum.KEY_CONTAINS.getCode().equals(filter.getFilterType())
+            && ValidateUtils.isBlank(filter.getFilterCompareKey())) {
+            return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "key包含的方式过滤，必须有过滤的key");
+        }
+
+        // value包含过滤
+        if (KafkaConsumerFilterEnum.VALUE_CONTAINS.getCode().equals(filter.getFilterType())
+            && ValidateUtils.isBlank(filter.getFilterCompareValue())) {
+            return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "value包含的方式过滤，必须有过滤的value");
+        }
+
         // 不包含过滤
         if (KafkaConsumerFilterEnum.NOT_CONTAINS.getCode().equals(filter.getFilterType())
                 && ValidateUtils.isBlank(filter.getFilterCompareKey()) && ValidateUtils.isBlank(filter.getFilterCompareValue())) {
@@ -550,6 +562,18 @@ public class KafkaClientTestManagerImpl implements KafkaClientTestManager {
             return true;
         }
 
+        // key包含过滤
+        if (KafkaConsumerFilterEnum.KEY_CONTAINS.getCode().equals(filter.getFilterType())
+            && (!ValidateUtils.isBlank(filter.getFilterCompareKey()) && consumerRecord.key() != null && consumerRecord.key().toString().contains(filter.getFilterCompareKey()))) {
+            return true;
+        }
+
+        // value包含过滤
+        if (KafkaConsumerFilterEnum.VALUE_CONTAINS.getCode().equals(filter.getFilterType())
+            && (!ValidateUtils.isBlank(filter.getFilterCompareValue()) && consumerRecord.value() != null && consumerRecord.value().toString().contains(filter.getFilterCompareValue()))) {
+            return true;
+        }
+
         // 不包含过滤
         if (KafkaConsumerFilterEnum.NOT_CONTAINS.getCode().equals(filter.getFilterType())
                 && (!ValidateUtils.isBlank(filter.getFilterCompareKey()) && (consumerRecord.key() == null || !consumerRecord.key().toString().contains(filter.getFilterCompareKey())))

km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/bean/dto/KafkaConsumerFilterDTO.java

@@ -19,7 +19,7 @@ public class KafkaConsumerFilterDTO extends BaseDTO {
     /**
      * @see KafkaConsumerFilterEnum
      */
-    @Range(min = 0, max = 5, message = "filterType最大和最小值必须在[0, 5]之间")
+    @Range(min = 0, max = 7, message = "filterType最大和最小值必须在[0, 7]之间")
     @ApiModelProperty(value = "开始消费位置的类型", example = "2")
     private Integer filterType;
 

km-enterprise/km-testing/src/main/java/com/xiaojukeji/know/streaming/km/testing/common/enums/KafkaConsumerFilterEnum.java

@@ -22,6 +22,10 @@ public enum KafkaConsumerFilterEnum {
 
     UNDER_SIZE(5, "size小于"),
 
+    KEY_CONTAINS(6, "key包含"),
+
+    VALUE_CONTAINS(7, "value包含"),
+
     ;
 
     private final Integer code;

km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggListMetrics

@@ -17,6 +17,13 @@
             }
           }
         },
+        {
+          "term": {
+            "brokerAgg" : {
+              "value": "1"
+            }
+          }
+        },
         {
           "range": {
             "timestamp": {

km-rest/pom.xml

@@ -143,6 +143,12 @@
             <version>${springboot.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.ulisesbocchio</groupId>
+            <artifactId>jasypt-spring-boot-starter</artifactId>
+            <version>3.0.5</version>
+        </dependency>
+
         <!--testcontainers-->
         <dependency>
             <groupId>org.testcontainers</groupId>

pom.xml

@@ -15,7 +15,7 @@
     </parent>
 
     <properties>
-        <revision>demo-3.4.0</revision>
+        <revision>enterprise-3.4.0</revision>
 
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>