验证功能是否正常 (#1193)

This commit is contained in:
EricZeng
2023-11-27 13:51:21 +08:00
committed by GitHub
17 changed files with 234 additions and 155 deletions

View File

@@ -146,7 +146,7 @@ PS: 提问请尽量把问题一次性描述清楚,并告知环境信息情况
**`2、微信群`**
微信加群:添加`mike_zhangliang``PenceXie``szzdzhp001`的微信号备注KnowStreaming加群。
微信加群:添加`PenceXie``szzdzhp001`的微信号备注KnowStreaming加群。
<br/>
加群之前有劳点一下 star一个小小的 star 是对KnowStreaming作者们努力建设社区的动力。

View File

@@ -7,7 +7,7 @@
### 3.3.1、Cluster 指标
| 指标名称 | 指标单位 | 指标含义 | kafka 版本 | 企业/开源版指标 |
| ------------------------- | -------- | ------------------------------------ | ---------------- | --------------- |
| ------------------------- | -------- |--------------------------------| ---------------- | --------------- |
| HealthScore | 分 | 集群总体的健康分 | 全部版本 | 开源版 |
| HealthCheckPassed | 个 | 集群总体健康检查通过数 | 全部版本 | 开源版 |
| HealthCheckTotal | 个 | 集群总体健康检查总数 | 全部版本 | 开源版 |
@@ -42,7 +42,7 @@
| PartitionMinISR_S | 个 | 集群中的小于 PartitionMinISR 总数 | 全部版本 | 开源版 |
| PartitionMinISR_E | 个 | 集群中的等于 PartitionMinISR 总数 | 全部版本 | 开源版 |
| PartitionURP | 个 | 集群中的未同步的 Partition 总数 | 全部版本 | 开源版 |
| MessagesIn | 条/s | 集群每消息写入条数 | 全部版本 | 开源版 |
| MessagesIn | 条/s | 集群每消息写入条数 | 全部版本 | 开源版 |
| Messages | 条 | 集群总的消息条数 | 全部版本 | 开源版 |
| LeaderMessages | 条 | 集群中 leader 总的消息条数 | 全部版本 | 开源版 |
| BytesIn | byte/s | 集群的每秒写入字节数 | 全部版本 | 开源版 |

View File

@@ -22,6 +22,7 @@
- [16、JMX连接失败怎么办](#16jmx连接失败怎么办)
- [17、zk监控无数据问题](#17zk监控无数据问题)
- [18、启动失败报NoClassDefFoundError如何解决](#18启动失败报noclassdeffounderror如何解决)
- [19、依赖ElasticSearch 8.0以上版本部署后指标信息无法正常显示如何解决]
## 1、支持哪些 Kafka 版本?
@@ -306,4 +307,15 @@ at org.springframework.beans.factory.support.ConstructorResolver.instantiate(Con
如果在在 `Windows``Mac``CentOS` 这几个操作系统下也出现了启动失败的问题可以重试2-3次看是否还是启动失败或者换一台机器试试。
## 依赖ElasticSearch 8.0以上版本部署后指标信息无法正常显示如何解决
**错误现象**
```log
Warnings: [299 Elasticsearch-8.9.1-a813d015ef1826148d9d389bd1c0d781c6e349f0 "Legacy index templates are deprecated in favor of composable templates."]
```
**问题原因**
1. ES8.0和ES7.0版本存在Template模式的差异建议使用 /_index_template 端点来管理模板;
2. ES java client在此版本的行为很奇怪表现为读取数据为空
**解决方法**
修改`es_template_create.sh`脚本中所有的`/_template``/_index_template`后执行即可。

View File

@@ -16,6 +16,13 @@ const babelOptions = {
cacheDirectory: true,
babelrc: false,
presets: [require.resolve('@babel/preset-env'), require.resolve('@babel/preset-typescript'), require.resolve('@babel/preset-react')],
overrides: [
// TODO编译时需要做的事情更多应该只针对目标第三方库
{
include: './node_modules',
sourceType: 'unambiguous'
}
],
plugins: [
[require.resolve('@babel/plugin-proposal-decorators'), { legacy: true }],
[require.resolve('@babel/plugin-proposal-class-properties'), { loose: true }],

View File

@@ -108,6 +108,7 @@
"optimize-css-assets-webpack-plugin": "^5.0.1",
"prettier": "2.3.2",
"progress-bar-webpack-plugin": "^1.12.1",
"pubsub-js": "^1.9.4",
"query-string": "^7.0.1",
"react-refresh": "^0.10.0",
"react-router-dom": "5.2.1",

View File

@@ -49,7 +49,7 @@ const ConnectDetailCard = (props: { record: any }) => {
return (
<>
{
<span style={{ fontFamily: 'HelveticaNeue-Medium', fontSize: 32, color: '#212529' }}>
<span style={{ fontFamily: 'HelveticaNeue-Medium', fontSize: 28, color: '#212529' }}>
{Utils.firstCharUppercase(type) || '-'}
</span>
}
@@ -64,7 +64,7 @@ const ConnectDetailCard = (props: { record: any }) => {
return (
<>
{
<span style={{ fontFamily: 'HelveticaNeue-Medium', fontSize: 32, color: stateEnum[state].color }}>
<span style={{ fontFamily: 'HelveticaNeue-Medium', fontSize: 28, color: stateEnum[state].color }}>
{Utils.firstCharUppercase(state) || '-'}
</span>
}

View File

@@ -48,6 +48,12 @@ export enum ClustersPermissionMap {
CONNECTOR_DELETE = 'Connector-删除',
CONNECTOR_RESTART = 'Connector-重启',
CONNECTOR_STOP_RESUME = 'Connector-暂停&恢复',
// Security
SECURITY_ACL_ADD = 'Security-ACL新增',
SECURITY_ACL_DELETE = 'Security-ACL删除',
SECURITY_USER_ADD = 'Security-User新增',
SECURITY_USER_DELETE = 'Security-User删除',
SECURITY_USER_EDIT_PASSWORD = 'Security-User修改密码',
}
export interface PermissionNode {
@@ -98,9 +104,7 @@ const CommonConfig = () => {
clustersPermissions.childList.forEach((node: PermissionNode) => node.has && userPermissions.push(node.permissionName));
// 获取用户在系统管理拥有的权限
const configPermissions = userPermissionTree.find(
(sys: PermissionNode) => sys.permissionName === ClustersPermissionMap.SYS_MANAGE
);
const configPermissions = userPermissionTree.find((sys: PermissionNode) => sys.permissionName === ClustersPermissionMap.SYS_MANAGE);
configPermissions &&
configPermissions.childList.forEach((node: PermissionNode) => node.has && userPermissions.push(node.permissionName));

View File

@@ -1,7 +1,7 @@
import api from '@src/api';
import CodeMirrorFormItem from '@src/components/CodeMirrorFormItem';
import customMessage from '@src/components/Message';
import { Button, Divider, Drawer, Form, message, Space, Utils } from 'knowdesign';
import { Button, Divider, Drawer, Form, message, Space, Utils, Select } from 'knowdesign';
import React, { forwardRef, useEffect, useImperativeHandle, useState } from 'react';
import { useParams } from 'react-router-dom';
import { ConnectCluster, ConnectorPlugin, ConnectorPluginConfig, OperateInfo } from './AddConnector';
@@ -9,9 +9,8 @@ import { ConnectCluster, ConnectorPlugin, ConnectorPluginConfig, OperateInfo } f
const PLACEHOLDER = `配置格式如下
{
"connectClusterName": "", // Connect Cluster 名称
"name": "", // Connect Cluster 名称
"config": { // 具体配置项
"name": "",
"connector.class": "",
"tasks.max": 1,
...
@@ -43,11 +42,16 @@ export default forwardRef((props: any, ref) => {
const onOpen = (type: 'create' | 'edit', connectClusterName?: string, defaultConfigs?: { [key: string]: any }) => {
if (defaultConfigs) {
setDefaultConfigs({ ...defaultConfigs, connectClusterName });
const connectorName = connectClusterName;
const connectClusterId = connectClusters.find((cluster) => cluster.label === connectClusterName).value;
form.setFieldsValue({
connectClusterId,
connectorName,
configs: JSON.stringify(
{
connectClusterName,
config: defaultConfigs,
// connectClusterName,
name: defaultConfigs.name,
config: { ...defaultConfigs, name: undefined },
},
null,
2
@@ -63,10 +67,11 @@ export default forwardRef((props: any, ref) => {
form.validateFields().then(
(data) => {
const postData = JSON.parse(data.configs);
postData.connectorName = postData.config.name;
postData.connectClusterId = connectClusters.find((cluster) => cluster.label === postData.connectClusterName).value;
delete postData.connectClusterName;
postData.connectorName = postData.name;
postData.connectClusterId = data.connectClusterId;
postData.config.name = postData.name;
// delete postData.connectClusterName;
delete postData.name;
Object.entries(postData.config).forEach(([key, val]) => {
if (val === null) {
delete postData.config[key];
@@ -161,6 +166,26 @@ export default forwardRef((props: any, ref) => {
}
>
<Form form={form} layout="vertical">
<Form.Item
name="connectClusterId"
label="Connect 集群"
rules={[
{
required: true,
validator(rule, value) {
if (!value) {
return Promise.reject('Connect 集群不能为空');
} else {
return Promise.resolve();
}
},
},
]}
initialValue={defaultConfigs?.connectClusterId}
className="connector-json-connectCluster"
>
<Select options={connectClusters} placeholder="请选择 Connect 集群" disabled={type === 'edit'} />
</Form.Item>
<Form.Item
name="configs"
validateTrigger="onBlur"
@@ -175,40 +200,27 @@ export default forwardRef((props: any, ref) => {
if (typeof v !== 'object') {
return Promise.reject('输入内容必须为 JSON');
}
let connectClusterId = -1;
// 校验 connectClusterName 字段
if (!v.connectClusterName) {
return Promise.reject('内容缺少 connectClusterName 字段或字段内容为空');
let connectClusterId = form.getFieldValue('connectClusterId');
// 校验 connectorName 字段
if (!v.name) {
return Promise.reject('内容缺少 name 项');
} else {
if (type === 'edit') {
if (v.connectClusterName !== defaultConfigs.connectClusterName) {
return Promise.reject('编辑模式下不允许修改 connectClusterName 字段');
}
} else {
if (!connectClusters.length) {
getConnectClusters();
return Promise.reject('connectClusterName 列表获取失败,请重试');
}
const targetConnectCluster = connectClusters.find((cluster) => cluster.label === v.connectClusterName);
if (!targetConnectCluster) {
return Promise.reject('connectClusterName 不存在,请检查');
} else {
connectClusterId = targetConnectCluster.value;
}
if (type === 'edit' && v.name !== defaultConfigs.name) {
return Promise.reject('编辑模式下不允许修改 name 字段');
}
}
if (!v.config || typeof v.config !== 'object') {
return Promise.reject('内容缺少 config 字段或字段格式错误');
} else {
// 校验 connectorName 字段
if (!v.config.name) {
return Promise.reject('config 字段下缺少 name 项');
} else {
if (type === 'edit' && v.config.name !== defaultConfigs.name) {
return Promise.reject('编辑模式下不允许修改 name 字段');
}
}
// // 校验 connectorName 字段
// if (!v.config.name) {
// return Promise.reject('config 字段下缺少 name 项');
// } else {
// if (type === 'edit' && v.config.name !== defaultConfigs.name) {
// return Promise.reject('编辑模式下不允许修改 name 字段');
// }
// }
if (!v.config['connector.class']) {
return Promise.reject('config 字段下缺少 connector.class 项');
} else if (type === 'edit' && v.config['connector.class'] !== defaultConfigs['connector.class']) {
@@ -217,6 +229,10 @@ export default forwardRef((props: any, ref) => {
}
if (type === 'create') {
// 校验创建时是否选择了connect集群
if (!connectClusterId) {
return Promise.reject('请先选择 Connect 集群');
}
// 异步校验 connector 名称是否重复 以及 className 是否存在
return Promise.all([
Utils.request(api.isConnectorExist(connectClusterId, v.config.name)),

View File

@@ -315,6 +315,7 @@ export const getWorkersColumns = (arg?: any) => {
// Detail
export const getConnectorsDetailColumns = (arg?: any) => {
const [global] = AppContainer.useGlobalValue();
const columns = [
{
title: 'Task ID',
@@ -363,6 +364,7 @@ export const getConnectorsDetailColumns = (arg?: any) => {
render: (_t: any, r: any) => {
return (
<div>
{global.hasPermission(ClustersPermissionMap.CONNECTOR_RESTART) ? (
<Popconfirm
title="是否重试当前任务?"
onConfirm={() => arg?.retryOption(r.taskId)}
@@ -373,6 +375,9 @@ export const getConnectorsDetailColumns = (arg?: any) => {
>
<a></a>
</Popconfirm>
) : (
<></>
)}
</div>
);
},

View File

@@ -185,9 +185,10 @@
.operate-connector-drawer-use-json {
.CodeMirror.cm-s-default {
height: calc(100vh - 146px);
height: calc(100vh - 196px);
}
.dcloud-form-item {
margin-top: 16px;
margin-bottom: 0 !important;
}
}

View File

@@ -4,7 +4,7 @@ import { useParams } from 'react-router-dom';
import EditTable from '../TestingProduce/component/EditTable';
import Api from '@src/api/index';
import moment from 'moment';
import PubSub from 'pubsub-js'
import PubSub from 'pubsub-js';
const CustomSelectResetTime = (props: { value?: string; onChange?: (val: Number | String) => void }) => {
const { value, onChange } = props;

View File

@@ -13,7 +13,7 @@ const carouselList = [
<img className="carousel-eg-ctr-two-img img-one" src={egTwoContent} />
<div className="carousel-eg-ctr-two-desc desc-one">
<span>Github: </span>
<span>5.8K</span>
<span>6.8K</span>
<span>+ Star的的实时流处理平台</span>
</div>
<div className="carousel-eg-ctr-two-desc desc-two">

View File

@@ -185,7 +185,7 @@
.operate-connector-drawer-use-json {
.CodeMirror.cm-s-default {
height: calc(100vh - 146px);
height: calc(100vh - 196px);
}
.dcloud-form-item {
margin-bottom: 0 !important;

View File

@@ -132,17 +132,35 @@ const AddDrawer = forwardRef((_, ref) => {
form.validateFields().then((formData) => {
const submitData = [];
const { configType, principle, kafkaUser } = formData;
if (configType === 'custom') {
// 1. 自定义权限
// TODO: 需要和后端联调
const { resourceType, resourcePatternType, aclPermissionType, aclOperation, aclClientHost } = formData;
const {
resourceType,
resourcePatternType,
aclPermissionType,
aclOperation,
aclClientHost,
cluster,
topicName,
topicPatternType,
groupName,
groupPatternType,
transactionalId,
transactionalIdPatternType,
} = formData;
submitData.push({
clusterId,
kafkaUser: principle === 'all' ? '*' : kafkaUser,
resourceType,
resourcePatternType,
resourceName: '*',
resourcePatternType: cluster
? 3
: topicPatternType
? topicPatternType
: groupPatternType
? groupPatternType
: transactionalIdPatternType,
resourceName: cluster ? cluster : topicName ? topicName : groupName ? groupName : transactionalId,
aclPermissionType,
aclOperation,
aclClientHost,
@@ -348,6 +366,7 @@ const AddDrawer = forwardRef((_, ref) => {
<Form.Item dependencies={[`${type}Principle`]} style={{ marginBottom: 0 }}>
{({ getFieldValue }) =>
getFieldValue(`${type}Principle`) === 'special' ? (
type !== 'transactionalId' ? (
<Form.Item
name={`${type}Name`}
dependencies={[`${type}PatternType`]}
@@ -379,6 +398,11 @@ const AddDrawer = forwardRef((_, ref) => {
placeholder={`请输入 ${type}Name`}
/>
</Form.Item>
) : (
<Form.Item name={`transactionalId`} rules={[{ required: true, message: `TransactionalId不能为空` }]}>
<Input placeholder={`请输入TransactionalId`}></Input>
</Form.Item>
)
) : null
}
</Form.Item>
@@ -400,7 +424,7 @@ const AddDrawer = forwardRef((_, ref) => {
<Radio value={ACL_PERMISSION_TYPE['Deny']}>Deny</Radio>
</Radio.Group>
</Form.Item>
<Form.Item
{/* <Form.Item
label="Pattern Type"
name="resourcePatternType"
rules={[{ required: true, message: 'Pattern Type 不能为空' }]}
@@ -410,7 +434,7 @@ const AddDrawer = forwardRef((_, ref) => {
<Radio value={ACL_PATTERN_TYPE['Literal']}>Literal</Radio>
<Radio value={ACL_PATTERN_TYPE['Prefixed']}>Prefixed</Radio>
</Radio.Group>
</Form.Item>
</Form.Item> */}
<Form.Item
label="Resource Type"
name="resourceType"
@@ -428,7 +452,7 @@ const AddDrawer = forwardRef((_, ref) => {
<Form.Item dependencies={['resourceType']}>
{({ getFieldValue }) => {
const type = getFieldValue('resourceType');
if (type === ACL_RESOURCE_TYPE['Cluster'] || type === ACL_RESOURCE_TYPE['TransactionalId']) {
if (type === ACL_RESOURCE_TYPE['Cluster']) {
//TODO需要和后端获取集群和事务接口联调
return (
<Form.Item
@@ -438,6 +462,8 @@ const AddDrawer = forwardRef((_, ref) => {
<Input placeholder={`请输入${type === 4 ? 'Cluster名称' : 'TransactionalId'}`}></Input>
</Form.Item>
);
} else if (type === ACL_RESOURCE_TYPE['TransactionalId']) {
return <PatternTypeFormItems type="transactionalId" />;
} else if (type === ACL_RESOURCE_TYPE['Topic']) {
return <PatternTypeFormItems type="topic" />;
} else if (type === ACL_RESOURCE_TYPE['Group']) {

View File

@@ -14,6 +14,7 @@ import AddACLDrawer, {
RESOURCE_TO_OPERATIONS_MAP,
RESOURCE_MAP_KEYS,
} from './EditDrawer';
import { ClustersPermissionMap } from '../CommonConfig';
import './index.less';
const { confirm } = Modal;
@@ -105,7 +106,7 @@ const SecurityACLs = (): JSX.Element => {
};
const columns = () => {
const baseColumns = [
const baseColumns: any = [
{
title: 'Principal',
dataIndex: 'kafkaUser',
@@ -143,7 +144,9 @@ const SecurityACLs = (): JSX.Element => {
title: 'Host',
dataIndex: 'aclClientHost',
},
{
];
if (global.hasPermission && global.hasPermission(ClustersPermissionMap.SECURITY_ACL_DELETE)) {
baseColumns.push({
title: '操作',
dataIndex: '',
width: 120,
@@ -156,8 +159,8 @@ const SecurityACLs = (): JSX.Element => {
</>
);
},
},
];
});
}
return baseColumns;
};
@@ -238,6 +241,7 @@ const SecurityACLs = (): JSX.Element => {
</Form.Item>
</Form>
</div>
{global.hasPermission && global.hasPermission(ClustersPermissionMap.SECURITY_ACL_ADD) ? (
<div className={`${tableHeaderPrefix}-right`}>
<Button
type="primary"
@@ -247,6 +251,9 @@ const SecurityACLs = (): JSX.Element => {
ACL
</Button>
</div>
) : (
<></>
)}
</div>
<ProTable
tableProps={{

View File

@@ -82,7 +82,7 @@ export const getTopicMessagesColmns = () => {
dataIndex: 'offset',
key: 'offset',
sorter: true,
render: (t: number) => (+t ? t.toLocaleString() : '-'),
render: (t: number) => (+t || +t === 0 ? t.toLocaleString() : '-'), // TODO: 千分位展示
},
{
title: 'Timestamp',

View File

@@ -299,7 +299,7 @@ public class ClusterMetricVersionItems extends BaseMetricVersionMetric {
// MessagesIn 指标
itemList.add( buildAllVersionsItem()
.name(CLUSTER_METRIC_MESSAGES_IN).unit("条/s").desc("集群每消息写入条数").category(CATEGORY_CLUSTER)
.name(CLUSTER_METRIC_MESSAGES_IN).unit("条/s").desc("集群每消息写入条数").category(CATEGORY_CLUSTER)
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
.jmxObjectName( JMX_SERVER_BROKER_MESSAGES_IN ).jmxAttribute(RATE_MIN_1)));