mirror of
https://github.com/didi/KnowStreaming.git
synced 2025-12-24 03:42:07 +08:00
526 lines
31 KiB
Markdown
526 lines
31 KiB
Markdown
## 6.2、版本升级手册
|
||
|
||
注意：
|
||
- 如果想升级至具体版本，需要将你当前版本至你期望使用版本的变更统统执行一遍，然后才能正常使用。
|
||
- 如果中间某个版本没有升级信息，则表示该版本直接替换安装包即可从前一个版本升级至当前版本。
|
||
|
||
### 升级至 `master` 版本
|
||
|
||
暂无
|
||
|
||
---
|
||
|
||
### 升级至 `3.4.0` 版本
|
||
|
||
**配置变更**
|
||
|
||
```yaml
|
||
# 新增的配置
|
||
request: # 请求相关的配置
|
||
api-call: # api调用
|
||
timeout-unit-ms: 8000 # 超时时间，默认8000毫秒
|
||
```
|
||
|
||
**SQL 变更**
|
||
```sql
|
||
-- 多集群管理权限2023-06-27新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2026', 'Connector-新增', '1593', '1', '2', 'Connector-新增', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2028', 'Connector-编辑', '1593', '1', '2', 'Connector-编辑', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2030', 'Connector-删除', '1593', '1', '2', 'Connector-删除', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2032', 'Connector-重启', '1593', '1', '2', 'Connector-重启', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2034', 'Connector-暂停&恢复', '1593', '1', '2', 'Connector-暂停&恢复', '0', 'know-streaming');
|
||
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2026', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2028', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2030', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2032', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2034', '0', 'know-streaming');
|
||
|
||
|
||
-- 多集群管理权限2023-06-29新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2036', 'Security-ACL新增', '1593', '1', '2', 'Security-ACL新增', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2038', 'Security-ACL删除', '1593', '1', '2', 'Security-ACL删除', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2040', 'Security-User新增', '1593', '1', '2', 'Security-User新增', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2042', 'Security-User删除', '1593', '1', '2', 'Security-User删除', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2044', 'Security-User修改密码', '1593', '1', '2', 'Security-User修改密码', '0', 'know-streaming');
|
||
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2036', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2038', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2040', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2042', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2044', '0', 'know-streaming');
|
||
|
||
|
||
-- 多集群管理权限2023-07-06新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2046', 'Group-删除', '1593', '1', '2', 'Group-删除', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2048', 'GroupOffset-Topic纬度删除', '1593', '1', '2', 'GroupOffset-Topic纬度删除', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2050', 'GroupOffset-Partition纬度删除', '1593', '1', '2', 'GroupOffset-Partition纬度删除', '0', 'know-streaming');
|
||
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2046', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2048', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2050', '0', 'know-streaming');
|
||
|
||
|
||
-- 多集群管理权限2023-07-18新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2052', 'Security-User查看密码', '1593', '1', '2', 'Security-User查看密码', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2052', '0', 'know-streaming');
|
||
```
|
||
|
||
---
|
||
|
||
### 升级至 `3.3.0` 版本
|
||
|
||
**SQL 变更**
|
||
```sql
|
||
ALTER TABLE `logi_security_user`
|
||
CHANGE COLUMN `phone` `phone` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'mobile' ;
|
||
|
||
ALTER TABLE ks_kc_connector ADD `heartbeat_connector_name` varchar(512) DEFAULT '' COMMENT '心跳检测connector名称';
|
||
ALTER TABLE ks_kc_connector ADD `checkpoint_connector_name` varchar(512) DEFAULT '' COMMENT '进度确认connector名称';
|
||
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_TOTAL_RECORD_ERRORS', '{\"value\" : 1}', 'MirrorMaker消息处理错误的次数', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_REPLICATION_LATENCY_MS_MAX', '{\"value\" : 6000}', 'MirrorMaker消息复制最大延迟时间', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_UNASSIGNED_TASK_COUNT', '{\"value\" : 20}', 'MirrorMaker未被分配的任务数量', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_FAILED_TASK_COUNT', '{\"value\" : 10}', 'MirrorMaker失败状态的任务数量', 'admin');
|
||
|
||
|
||
-- 多集群管理权限2023-01-05新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2012', 'Topic-新增Topic复制', '1593', '1', '2', 'Topic-新增Topic复制', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2014', 'Topic-详情-取消Topic复制', '1593', '1', '2', 'Topic-详情-取消Topic复制', '0', 'know-streaming');
|
||
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2012', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2014', '0', 'know-streaming');
|
||
|
||
|
||
-- 多集群管理权限2023-01-18新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2016', 'MM2-新增', '1593', '1', '2', 'MM2-新增', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2018', 'MM2-编辑', '1593', '1', '2', 'MM2-编辑', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2020', 'MM2-删除', '1593', '1', '2', 'MM2-删除', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2022', 'MM2-重启', '1593', '1', '2', 'MM2-重启', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2024', 'MM2-暂停&恢复', '1593', '1', '2', 'MM2-暂停&恢复', '0', 'know-streaming');
|
||
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2016', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2018', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2020', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2022', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2024', '0', 'know-streaming');
|
||
|
||
|
||
DROP TABLE IF EXISTS `ks_ha_active_standby_relation`;
|
||
CREATE TABLE `ks_ha_active_standby_relation` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
||
`active_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '主集群ID',
|
||
`standby_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '备集群ID',
|
||
`res_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '资源名称',
|
||
`res_type` int(11) NOT NULL DEFAULT '-1' COMMENT '资源类型，0：集群，1：镜像Topic，2：主备Topic',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_cluster_res` (`res_type`,`active_cluster_phy_id`,`standby_cluster_phy_id`,`res_name`),
|
||
UNIQUE KEY `uniq_res_type_standby_cluster_res_name` (`res_type`,`standby_cluster_phy_id`,`res_name`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='HA主备关系表';
|
||
|
||
|
||
-- 删除idx_cluster_phy_id 索引并新增idx_cluster_update_time索引
|
||
ALTER TABLE `ks_km_kafka_change_record` DROP INDEX `idx_cluster_phy_id` ,
|
||
ADD INDEX `idx_cluster_update_time` (`cluster_phy_id` ASC, `update_time` ASC);
|
||
```
|
||
|
||
---
|
||
|
||
### 升级至 `3.2.0` 版本
|
||
|
||
**配置变更**
|
||
|
||
```yaml
|
||
# 新增如下配置
|
||
|
||
spring:
|
||
logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
|
||
enable: true # true表示开启job任务, false表关闭。KS在部署上可以考虑部署两套服务，一套处理前端请求，一套执行job任务，此时可以通过该字段进行控制
|
||
|
||
# 线程池大小相关配置
|
||
thread-pool:
|
||
es:
|
||
search: # es查询线程池
|
||
thread-num: 20 # 线程池大小
|
||
queue-size: 10000 # 队列大小
|
||
|
||
# 客户端池大小相关配置
|
||
client-pool:
|
||
kafka-admin:
|
||
client-cnt: 1 # 每个Kafka集群创建的KafkaAdminClient数
|
||
|
||
# ES客户端配置
|
||
es:
|
||
index:
|
||
expire: 15 # 索引过期天数，15表示超过15天的索引会被KS过期删除
|
||
```
|
||
|
||
**SQL 变更**
|
||
```sql
|
||
DROP TABLE IF EXISTS `ks_kc_connect_cluster`;
|
||
CREATE TABLE `ks_kc_connect_cluster` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Connect集群ID',
|
||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
||
`name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群名称',
|
||
`group_name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群Group名称',
|
||
`cluster_url` varchar(1024) NOT NULL DEFAULT '' COMMENT '集群地址',
|
||
`member_leader_url` varchar(1024) NOT NULL DEFAULT '' COMMENT 'URL地址',
|
||
`version` varchar(64) NOT NULL DEFAULT '' COMMENT 'connect版本',
|
||
`jmx_properties` text COMMENT 'JMX配置',
|
||
`state` tinyint(4) NOT NULL DEFAULT '1' COMMENT '集群使用的消费组状态，也表示集群状态:-1 Unknown,0 ReBalance,1 Active,2 Dead,3 Empty',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '接入时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_id_group_name` (`id`,`group_name`),
|
||
UNIQUE KEY `uniq_name_kafka_cluster` (`name`,`kafka_cluster_phy_id`),
|
||
KEY `idx_kafka_cluster_phy_id` (`kafka_cluster_phy_id`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Connect集群信息表';
|
||
|
||
|
||
DROP TABLE IF EXISTS `ks_kc_connector`;
|
||
CREATE TABLE `ks_kc_connector` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
||
`connect_cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Connect集群ID',
|
||
`connector_name` varchar(512) NOT NULL DEFAULT '' COMMENT 'Connector名称',
|
||
`connector_class_name` varchar(512) NOT NULL DEFAULT '' COMMENT 'Connector类',
|
||
`connector_type` varchar(32) NOT NULL DEFAULT '' COMMENT 'Connector类型',
|
||
`state` varchar(45) NOT NULL DEFAULT '' COMMENT '状态',
|
||
`topics` text COMMENT '访问过的Topics',
|
||
`task_count` int(11) NOT NULL DEFAULT '0' COMMENT '任务数',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_connect_cluster_id_connector_name` (`connect_cluster_id`,`connector_name`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Connector信息表';
|
||
|
||
|
||
DROP TABLE IF EXISTS `ks_kc_worker`;
|
||
CREATE TABLE `ks_kc_worker` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
||
`connect_cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Connect集群ID',
|
||
`member_id` varchar(512) NOT NULL DEFAULT '' COMMENT '成员ID',
|
||
`host` varchar(128) NOT NULL DEFAULT '' COMMENT '主机名',
|
||
`jmx_port` int(16) NOT NULL DEFAULT '-1' COMMENT 'Jmx端口',
|
||
`url` varchar(1024) NOT NULL DEFAULT '' COMMENT 'URL信息',
|
||
`leader_url` varchar(1024) NOT NULL DEFAULT '' COMMENT 'leaderURL信息',
|
||
`leader` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1是leader，0不是leader',
|
||
`worker_id` varchar(128) NOT NULL COMMENT 'worker地址',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_cluster_id_member_id` (`connect_cluster_id`,`member_id`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker信息表';
|
||
|
||
|
||
DROP TABLE IF EXISTS `ks_kc_worker_connector`;
|
||
CREATE TABLE `ks_kc_worker_connector` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
||
`connect_cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Connect集群ID',
|
||
`connector_name` varchar(512) NOT NULL DEFAULT '' COMMENT 'Connector名称',
|
||
`worker_member_id` varchar(256) NOT NULL DEFAULT '',
|
||
`task_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'Task的ID',
|
||
`state` varchar(128) DEFAULT NULL COMMENT '任务状态',
|
||
`worker_id` varchar(128) DEFAULT NULL COMMENT 'worker信息',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_relation` (`connect_cluster_id`,`connector_name`,`task_id`,`worker_member_id`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Worker和Connector关系表';
|
||
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_CONNECTOR_FAILED_TASK_COUNT', '{\"value\" : 1}', 'connector失败状态的任务数量', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_CONNECTOR_UNASSIGNED_TASK_COUNT', '{\"value\" : 1}', 'connector未被分配的任务数量', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_CONNECT_CLUSTER_TASK_STARTUP_FAILURE_PERCENTAGE', '{\"value\" : 0.05}', 'Connect集群任务启动失败概率', 'admin');
|
||
```
|
||
|
||
---
|
||
|
||
### 升级至 `v3.1.0` 版本
|
||
|
||
```sql
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_BRAIN_SPLIT', '{ \"value\": 1} ', 'ZK 脑裂', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_OUTSTANDING_REQUESTS', '{ \"amount\": 100, \"ratio\":0.8} ', 'ZK Outstanding 请求堆积数', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_WATCH_COUNT', '{ \"amount\": 100000, \"ratio\": 0.8 } ', 'ZK WatchCount 数', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_ALIVE_CONNECTIONS', '{ \"amount\": 10000, \"ratio\": 0.8 } ', 'ZK 连接数', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_APPROXIMATE_DATA_SIZE', '{ \"amount\": 524288000, \"ratio\": 0.8 } ', 'ZK 数据大小(Byte)', 'admin');
|
||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_SENT_RATE', '{ \"amount\": 500000, \"ratio\": 0.8 } ', 'ZK 发包数', 'admin');
|
||
|
||
```
|
||
|
||
### 升级至 `v3.0.1` 版本
|
||
|
||
**ES 索引模版**
|
||
```bash
|
||
# 新增 ks_kafka_zookeeper_metric 索引模版。
|
||
# 可通过再次执行 bin/init_es_template.sh 脚本，创建该索引模版。
|
||
|
||
# 索引模版内容
|
||
PUT _template/ks_kafka_zookeeper_metric
|
||
{
|
||
"order" : 10,
|
||
"index_patterns" : [
|
||
"ks_kafka_zookeeper_metric*"
|
||
],
|
||
"settings" : {
|
||
"index" : {
|
||
"number_of_shards" : "10"
|
||
}
|
||
},
|
||
"mappings" : {
|
||
"properties" : {
|
||
"routingValue" : {
|
||
"type" : "text",
|
||
"fields" : {
|
||
"keyword" : {
|
||
"ignore_above" : 256,
|
||
"type" : "keyword"
|
||
}
|
||
}
|
||
},
|
||
"clusterPhyId" : {
|
||
"type" : "long"
|
||
},
|
||
"metrics" : {
|
||
"properties" : {
|
||
"AvgRequestLatency" : {
|
||
"type" : "double"
|
||
},
|
||
"MinRequestLatency" : {
|
||
"type" : "double"
|
||
},
|
||
"MaxRequestLatency" : {
|
||
"type" : "double"
|
||
},
|
||
"OutstandingRequests" : {
|
||
"type" : "double"
|
||
},
|
||
"NodeCount" : {
|
||
"type" : "double"
|
||
},
|
||
"WatchCount" : {
|
||
"type" : "double"
|
||
},
|
||
"NumAliveConnections" : {
|
||
"type" : "double"
|
||
},
|
||
"PacketsReceived" : {
|
||
"type" : "double"
|
||
},
|
||
"PacketsSent" : {
|
||
"type" : "double"
|
||
},
|
||
"EphemeralsCount" : {
|
||
"type" : "double"
|
||
},
|
||
"ApproximateDataSize" : {
|
||
"type" : "double"
|
||
},
|
||
"OpenFileDescriptorCount" : {
|
||
"type" : "double"
|
||
},
|
||
"MaxFileDescriptorCount" : {
|
||
"type" : "double"
|
||
}
|
||
}
|
||
},
|
||
"key" : {
|
||
"type" : "text",
|
||
"fields" : {
|
||
"keyword" : {
|
||
"ignore_above" : 256,
|
||
"type" : "keyword"
|
||
}
|
||
}
|
||
},
|
||
"timestamp" : {
|
||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||
"type" : "date"
|
||
}
|
||
}
|
||
},
|
||
"aliases" : { }
|
||
}
|
||
```
|
||
|
||
|
||
**SQL 变更**
|
||
|
||
```sql
|
||
DROP TABLE IF EXISTS `ks_km_zookeeper`;
|
||
CREATE TABLE `ks_km_zookeeper` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
||
`cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '物理集群ID',
|
||
`host` varchar(128) NOT NULL DEFAULT '' COMMENT 'zookeeper主机名',
|
||
`port` int(16) NOT NULL DEFAULT '-1' COMMENT 'zookeeper端口',
|
||
`role` varchar(16) NOT NULL DEFAULT '' COMMENT '角色, leader follower observer',
|
||
`version` varchar(128) NOT NULL DEFAULT '' COMMENT 'zookeeper版本',
|
||
`status` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1存活，0未存活，11存活但是4字命令使用不了',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_cluster_phy_id_host_port` (`cluster_phy_id`,`host`, `port`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Zookeeper信息表';
|
||
|
||
|
||
DROP TABLE IF EXISTS `ks_km_group`;
|
||
CREATE TABLE `ks_km_group` (
|
||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
||
`cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群id',
|
||
`name` varchar(192) COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Group名称',
|
||
`member_count` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '成员数',
|
||
`topic_members` text CHARACTER SET utf8 COMMENT 'group消费的topic列表',
|
||
`partition_assignor` varchar(255) CHARACTER SET utf8 NOT NULL COMMENT '分配策略',
|
||
`coordinator_id` int(11) NOT NULL COMMENT 'group协调器brokerId',
|
||
`type` int(11) NOT NULL COMMENT 'group类型 0：consumer 1：connector',
|
||
`state` varchar(64) CHARACTER SET utf8 NOT NULL DEFAULT '' COMMENT '状态',
|
||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
||
PRIMARY KEY (`id`),
|
||
UNIQUE KEY `uniq_cluster_phy_id_name` (`cluster_phy_id`,`name`)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Group信息表';
|
||
|
||
```
|
||
|
||
|
||
### 升级至 `v3.0.0` 版本
|
||
|
||
**SQL 变更**
|
||
|
||
```sql
|
||
ALTER TABLE `ks_km_physical_cluster`
|
||
ADD COLUMN `zk_properties` TEXT NULL COMMENT 'ZK配置' AFTER `jmx_properties`;
|
||
```
|
||
|
||
---
|
||
|
||
|
||
### 升级至 `v3.0.0-beta.2`版本
|
||
|
||
**配置变更**
|
||
|
||
```yaml
|
||
|
||
# 新增配置
|
||
spring:
|
||
logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
|
||
login-extend-bean-name: logiSecurityDefaultLoginExtendImpl # 使用的登录系统Service的Bean名称，无需修改
|
||
|
||
# 线程池大小相关配置，在task模块中，新增了三类线程池，
|
||
# 从而减少不同类型任务之间的相互影响，以及减少对logi-job内的线程池的影响
|
||
thread-pool:
|
||
task: # 任务模块的配置
|
||
metrics: # metrics采集任务配置
|
||
thread-num: 18 # metrics采集任务线程池核心线程数
|
||
queue-size: 180 # metrics采集任务线程池队列大小
|
||
metadata: # metadata同步任务配置
|
||
thread-num: 27 # metadata同步任务线程池核心线程数
|
||
queue-size: 270 # metadata同步任务线程池队列大小
|
||
common: # 剩余其他任务配置
|
||
thread-num: 15 # 剩余其他任务线程池核心线程数
|
||
queue-size: 150 # 剩余其他任务线程池队列大小
|
||
|
||
# 删除配置，下列配置将不再使用
|
||
thread-pool:
|
||
task: # 任务模块的配置
|
||
heaven: # 采集任务配置
|
||
thread-num: 20 # 采集任务线程池核心线程数
|
||
queue-size: 1000 # 采集任务线程池队列大小
|
||
|
||
```
|
||
|
||
**SQL 变更**
|
||
|
||
```sql
|
||
-- 多集群管理权限2022-09-06新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2000', '多集群管理查看', '1593', '1', '2', '多集群管理查看', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2002', 'Topic-迁移副本', '1593', '1', '2', 'Topic-迁移副本', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2004', 'Topic-扩缩副本', '1593', '1', '2', 'Topic-扩缩副本', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2006', 'Cluster-LoadReBalance-周期均衡', '1593', '1', '2', 'Cluster-LoadReBalance-周期均衡', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2008', 'Cluster-LoadReBalance-立即均衡', '1593', '1', '2', 'Cluster-LoadReBalance-立即均衡', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2010', 'Cluster-LoadReBalance-设置集群规格', '1593', '1', '2', 'Cluster-LoadReBalance-设置集群规格', '0', 'know-streaming');
|
||
|
||
|
||
-- 系统管理权限2022-09-06新增
|
||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('3000', '系统管理查看', '1595', '1', '2', '系统管理查看', '0', 'know-streaming');
|
||
|
||
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2000', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2002', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2004', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2006', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2008', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2010', '0', 'know-streaming');
|
||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '3000', '0', 'know-streaming');
|
||
|
||
-- 修改字段长度
|
||
ALTER TABLE `logi_security_oplog`
|
||
CHANGE COLUMN `operator_ip` `operator_ip` VARCHAR(64) NOT NULL COMMENT '操作者ip' ,
|
||
CHANGE COLUMN `operator` `operator` VARCHAR(64) NULL DEFAULT NULL COMMENT '操作者账号' ,
|
||
CHANGE COLUMN `operate_page` `operate_page` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '操作页面' ,
|
||
CHANGE COLUMN `operate_type` `operate_type` VARCHAR(64) NOT NULL COMMENT '操作类型' ,
|
||
CHANGE COLUMN `target_type` `target_type` VARCHAR(64) NOT NULL COMMENT '对象分类' ,
|
||
CHANGE COLUMN `target` `target` VARCHAR(1024) NOT NULL COMMENT '操作对象' ,
|
||
CHANGE COLUMN `operation_methods` `operation_methods` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '操作方式' ;
|
||
```
|
||
|
||
---
|
||
|
||
### 升级至 `v3.0.0-beta.1`版本
|
||
|
||
**SQL 变更**
|
||
|
||
1、在`ks_km_broker`表增加了一个监听信息字段。
|
||
2、为`logi_security_oplog`表 operation_methods 字段设置默认值''。
|
||
因此需要执行下面的 sql 对数据库表进行更新。
|
||
|
||
```sql
|
||
ALTER TABLE `ks_km_broker`
|
||
ADD COLUMN `endpoint_map` VARCHAR(1024) NOT NULL DEFAULT '' COMMENT '监听信息' AFTER `update_time`;
|
||
|
||
ALTER TABLE `logi_security_oplog`
|
||
ALTER COLUMN `operation_methods` set default '';
|
||
|
||
```
|
||
|
||
---
|
||
|
||
### `2.x`版本 升级至 `v3.0.0-beta.0`版本
|
||
|
||
**升级步骤：**
|
||
|
||
1. 依旧使用**`2.x 版本的 DB`**，在上面初始化 3.0.0 版本所需数据库表结构及数据；
|
||
2. 将 2.x 版本中的集群，在 3.0.0 版本，手动逐一接入；
|
||
3. 将 Topic 业务数据，迁移至 3.0.0 表中，详见下方 SQL；
|
||
|
||
**注意事项**
|
||
|
||
- 建议升级 3.0.0 版本过程中，保留 2.x 版本的使用，待 3.0.0 版本稳定使用后，再下线 2.x 版本；
|
||
- 3.0.0 版本仅需要`集群信息`及`Topic的描述信息`。2.x 版本的 DB 的其他数据 3.0.0 版本都不需要；
|
||
- 部署 3.0.0 版本之后，集群、Topic 等指标数据都为空，3.0.0 版本会周期进行采集，运行一段时间之后就会有该数据了，因此不会将 2.x 中的指标数据进行迁移；
|
||
|
||
**迁移数据**
|
||
|
||
```sql
|
||
-- 迁移Topic的备注信息。
|
||
-- 需在 3.0.0 部署完成后，再执行该SQL。
|
||
-- 考虑到 2.x 版本中还存在增量数据，因此建议改SQL周期执行，是的增量数据也能被迁移至 3.0.0 版本中。
|
||
|
||
UPDATE ks_km_topic
|
||
INNER JOIN
|
||
(SELECT
|
||
topic.cluster_id AS cluster_id,
|
||
topic.topic_name AS topic_name,
|
||
topic.description AS description
|
||
FROM topic WHERE description != ''
|
||
) AS t
|
||
|
||
ON ks_km_topic.cluster_phy_id = t.cluster_id
|
||
AND ks_km_topic.topic_name = t.topic_name
|
||
AND ks_km_topic.id > 0
|
||
SET ks_km_topic.description = t.description;
|
||
``` |