mirror of
https://github.com/didi/KnowStreaming.git
synced 2025-12-25 04:32:12 +08:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
66e3da5d2f | ||
|
|
5f2adfe74e | ||
|
|
cecfde906a |
43
.github/workflows/ci_build.yml
vendored
43
.github/workflows/ci_build.yml
vendored
@@ -1,43 +0,0 @@
|
|||||||
name: KnowStreaming Build
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [ "*" ]
|
|
||||||
pull_request:
|
|
||||||
branches: [ "*" ]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Set up JDK 11
|
|
||||||
uses: actions/setup-java@v3
|
|
||||||
with:
|
|
||||||
java-version: '11'
|
|
||||||
distribution: 'temurin'
|
|
||||||
cache: maven
|
|
||||||
|
|
||||||
- name: Setup Node
|
|
||||||
uses: actions/setup-node@v1
|
|
||||||
with:
|
|
||||||
node-version: '12.22.12'
|
|
||||||
|
|
||||||
- name: Build With Maven
|
|
||||||
run: mvn -Prelease-package -Dmaven.test.skip=true clean install -U
|
|
||||||
|
|
||||||
- name: Get KnowStreaming Version
|
|
||||||
if: ${{ success() }}
|
|
||||||
run: |
|
|
||||||
version=`mvn -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive exec:exec -q`
|
|
||||||
echo "VERSION=${version}" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Upload Binary Package
|
|
||||||
if: ${{ success() }}
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: KnowStreaming-${{ env.VERSION }}.tar.gz
|
|
||||||
path: km-dist/target/KnowStreaming-${{ env.VERSION }}.tar.gz
|
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
|
|
||||||
# Contributor Covenant Code of Conduct
|
|
||||||
|
|
||||||
## Our Pledge
|
|
||||||
|
|
||||||
In the interest of fostering an open and welcoming environment, we as
|
|
||||||
contributors and maintainers pledge to making participation in our project, and
|
|
||||||
our community a harassment-free experience for everyone, regardless of age, body
|
|
||||||
size, disability, ethnicity, gender identity and expression, level of experience,
|
|
||||||
education, socio-economic status, nationality, personal appearance, race,
|
|
||||||
religion, or sexual identity and orientation.
|
|
||||||
|
|
||||||
## Our Standards
|
|
||||||
|
|
||||||
Examples of behavior that contributes to creating a positive environment
|
|
||||||
include:
|
|
||||||
|
|
||||||
* Using welcoming and inclusive language
|
|
||||||
* Being respectful of differing viewpoints and experiences
|
|
||||||
* Gracefully accepting constructive criticism
|
|
||||||
* Focusing on what is best for the community
|
|
||||||
* Showing empathy towards other community members
|
|
||||||
|
|
||||||
Examples of unacceptable behavior by participants include:
|
|
||||||
|
|
||||||
* The use of sexualized language or imagery and unwelcome sexual attention or
|
|
||||||
advances
|
|
||||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
|
||||||
* Public or private harassment
|
|
||||||
* Publishing others' private information, such as a physical or electronic
|
|
||||||
address, without explicit permission
|
|
||||||
* Other conduct which could reasonably be considered inappropriate in a
|
|
||||||
professional setting
|
|
||||||
|
|
||||||
## Our Responsibilities
|
|
||||||
|
|
||||||
Project maintainers are responsible for clarifying the standards of acceptable
|
|
||||||
behavior and are expected to take appropriate and fair corrective action in
|
|
||||||
response to any instances of unacceptable behavior.
|
|
||||||
|
|
||||||
Project maintainers have the right and responsibility to remove, edit, or
|
|
||||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
|
||||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
|
||||||
permanently any contributor for other behaviors that they deem inappropriate,
|
|
||||||
threatening, offensive, or harmful.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
This Code of Conduct applies both within project spaces and in public spaces
|
|
||||||
when an individual is representing the project or its community. Examples of
|
|
||||||
representing a project or community include using an official project e-mail
|
|
||||||
address, posting via an official social media account, or acting as an appointed
|
|
||||||
representative at an online or offline event. Representation of a project may be
|
|
||||||
further defined and clarified by project maintainers.
|
|
||||||
|
|
||||||
## Enforcement
|
|
||||||
|
|
||||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
|
||||||
reported by contacting the project team at https://knowstreaming.com/support-center . All
|
|
||||||
complaints will be reviewed and investigated and will result in a response that
|
|
||||||
is deemed necessary and appropriate to the circumstances. The project team is
|
|
||||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
|
||||||
Further details of specific enforcement policies may be posted separately.
|
|
||||||
|
|
||||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
|
||||||
faith may face temporary or permanent repercussions as determined by other
|
|
||||||
members of the project's leadership.
|
|
||||||
|
|
||||||
## Attribution
|
|
||||||
|
|
||||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
|
||||||
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
|
||||||
|
|
||||||
[homepage]: https://www.contributor-covenant.org
|
|
||||||
150
CONTRIBUTING.md
150
CONTRIBUTING.md
@@ -1,150 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
# 为KnowStreaming做贡献
|
|
||||||
|
|
||||||
|
|
||||||
欢迎👏🏻来到KnowStreaming！本文档是关于如何为KnowStreaming做出贡献的指南。
|
|
||||||
|
|
||||||
如果您发现不正确或遗漏的内容, 请留下意见/建议。
|
|
||||||
|
|
||||||
## 行为守则
|
|
||||||
请务必阅读并遵守我们的 [行为准则](./CODE_OF_CONDUCT.md).
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 贡献
|
|
||||||
|
|
||||||
**KnowStreaming** 欢迎任何角色的新参与者，包括 **User** 、**Contributor**、**Committer**、**PMC** 。
|
|
||||||
|
|
||||||
我们鼓励新人积极加入 **KnowStreaming** 项目，从User到Contributor、Committer ，甚至是 PMC 角色。
|
|
||||||
|
|
||||||
为了做到这一点，新人需要积极地为 **KnowStreaming** 项目做出贡献。以下介绍如何对 **KnowStreaming** 进行贡献。
|
|
||||||
|
|
||||||
|
|
||||||
### 创建/打开 Issue
|
|
||||||
|
|
||||||
如果您在文档中发现拼写错误、在代码中**发现错误**或想要**新功能**或想要**提供建议**，您可以在 GitHub 上[创建一个Issue](https://github.com/didi/KnowStreaming/issues/new/choose) 进行报告。
|
|
||||||
|
|
||||||
|
|
||||||
如果您想直接贡献, 您可以选择下面标签的问题。
|
|
||||||
|
|
||||||
- [contribution welcome](https://github.com/didi/KnowStreaming/labels/contribution%20welcome) : 非常需要解决/新增 的Issues
|
|
||||||
- [good first issue](https://github.com/didi/KnowStreaming/labels/good%20first%20issue): 对新人比较友好, 新人可以拿这个Issue来练练手热热身。
|
|
||||||
|
|
||||||
<font color=red ><b> 请注意，任何 PR 都必须与有效issue相关联。否则，PR 将被拒绝。</b></font>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 开始你的贡献
|
|
||||||
|
|
||||||
**分支介绍**
|
|
||||||
|
|
||||||
我们将 `dev`分支作为开发分支, 说明这是一个不稳定的分支。
|
|
||||||
|
|
||||||
此外,我们的分支模型符合 [https://nvie.com/posts/a-successful-git-branching-model/](https://nvie.com/posts/a-successful-git-branching-model/). 我们强烈建议新人在创建PR之前先阅读上述文章。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**贡献流程**
|
|
||||||
|
|
||||||
为方便描述,我们这里定义一下2个名词：
|
|
||||||
|
|
||||||
自己Fork出来的仓库是私人仓库, 我们这里称之为 ：**分叉仓库**
|
|
||||||
Fork的源项目,我们称之为：**源仓库**
|
|
||||||
|
|
||||||
|
|
||||||
现在，如果您准备好创建PR, 以下是贡献者的工作流程:
|
|
||||||
|
|
||||||
1. Fork [KnowStreaming](https://github.com/didi/KnowStreaming) 项目到自己的仓库
|
|
||||||
|
|
||||||
2. 从源仓库的`dev`拉取并创建自己的本地分支,例如: `dev`
|
|
||||||
3. 在本地分支上对代码进行修改
|
|
||||||
4. Rebase 开发分支, 并解决冲突
|
|
||||||
5. commit 并 push 您的更改到您自己的**分叉仓库**
|
|
||||||
6. 创建一个 Pull Request 到**源仓库**的`dev`分支中。
|
|
||||||
7. 等待回复。如果回复的慢，请无情的催促。
|
|
||||||
|
|
||||||
|
|
||||||
更为详细的贡献流程请看：[贡献流程](./docs/contributer_guide/贡献流程.md)
|
|
||||||
|
|
||||||
创建Pull Request时：
|
|
||||||
|
|
||||||
1. 请遵循 PR的 [模板](./.github/PULL_REQUEST_TEMPLATE.md)
|
|
||||||
2. 请确保 PR 有相应的issue。
|
|
||||||
3. 如果您的 PR 包含较大的更改，例如组件重构或新组件，请编写有关其设计和使用的详细文档(在对应的issue中)。
|
|
||||||
4. 注意单个 PR 不能太大。如果需要进行大量更改，最好将更改分成几个单独的 PR。
|
|
||||||
5. 在合并PR之前，尽量的将最终的提交信息清晰简洁, 将多次修改的提交尽可能的合并为一次提交。
|
|
||||||
6. 创建 PR 后，将为PR分配一个或多个reviewers。
|
|
||||||
|
|
||||||
|
|
||||||
<font color=red><b>如果您的 PR 包含较大的更改，例如组件重构或新组件，请编写有关其设计和使用的详细文档。</b></font>
|
|
||||||
|
|
||||||
|
|
||||||
# 代码审查指南
|
|
||||||
|
|
||||||
Commiter将轮流review代码，以确保在合并前至少有一名Commiter
|
|
||||||
|
|
||||||
一些原则：
|
|
||||||
|
|
||||||
- 可读性——重要的代码应该有详细的文档。API 应该有 Javadoc。代码风格应与现有风格保持一致。
|
|
||||||
- 优雅：新的函数、类或组件应该设计得很好。
|
|
||||||
- 可测试性——单元测试用例应该覆盖 80% 的新代码。
|
|
||||||
- 可维护性 - 遵守我们的编码规范。
|
|
||||||
|
|
||||||
|
|
||||||
# 开发者
|
|
||||||
|
|
||||||
## 成为Contributor
|
|
||||||
|
|
||||||
只要成功提交并合并PR , 则为Contributor
|
|
||||||
|
|
||||||
贡献者名单请看：[贡献者名单](./docs/contributer_guide/开发者名单.md)
|
|
||||||
|
|
||||||
## 尝试成为Commiter
|
|
||||||
|
|
||||||
一般来说, 贡献8个重要的补丁并至少让三个不同的人来Review他们(您需要3个Commiter的支持)。
|
|
||||||
然后请人给你提名, 您需要展示您的
|
|
||||||
|
|
||||||
1. 至少8个重要的PR和项目的相关问题
|
|
||||||
2. 与团队合作的能力
|
|
||||||
3. 了解项目的代码库和编码风格
|
|
||||||
4. 编写好代码的能力
|
|
||||||
|
|
||||||
当前的Commiter可以通过在KnowStreaming中的Issue标签 `nomination`(提名)来提名您
|
|
||||||
|
|
||||||
1. 你的名字和姓氏
|
|
||||||
2. 指向您的Git个人资料的链接
|
|
||||||
3. 解释为什么你应该成为Commiter
|
|
||||||
4. 详细说明提名人与您合作的3个PR以及相关问题,这些问题可以证明您的能力。
|
|
||||||
|
|
||||||
另外2个Commiter需要支持您的**提名**，如果5个工作日内没有人反对，您就是提交者,如果有人反对或者想要更多的信息，Commiter会讨论并通常达成共识(5个工作日内) 。
|
|
||||||
|
|
||||||
|
|
||||||
# 开源奖励计划
|
|
||||||
|
|
||||||
|
|
||||||
我们非常欢迎开发者们为KnowStreaming开源项目贡献一份力量，相应也将给予贡献者激励以表认可与感谢。
|
|
||||||
|
|
||||||
|
|
||||||
## 参与贡献
|
|
||||||
|
|
||||||
1. 积极参与 Issue 的讨论，如答疑解惑、提供想法或报告无法解决的错误（Issue）
|
|
||||||
2. 撰写和改进项目的文档（Wiki）
|
|
||||||
3. 提交补丁优化代码（Coding）
|
|
||||||
|
|
||||||
|
|
||||||
## 你将获得
|
|
||||||
|
|
||||||
1. 加入KnowStreaming开源项目贡献者名单并展示
|
|
||||||
2. KnowStreaming开源贡献者证书(纸质&电子版)
|
|
||||||
3. KnowStreaming贡献者精美大礼包(KnowStreamin/滴滴 周边)
|
|
||||||
|
|
||||||
|
|
||||||
## 相关规则
|
|
||||||
|
|
||||||
- Contributer和Commiter都会有对应的证书和对应的礼包
|
|
||||||
- 每季度有KnowStreaming项目团队评选出杰出贡献者,颁发相应证书。
|
|
||||||
- 年末进行年度评选
|
|
||||||
|
|
||||||
贡献者名单请看：[贡献者名单](./docs/contributer_guide/开发者名单.md)
|
|
||||||
BIN
KS-PRD-3.0-beta1.docx
Normal file
BIN
KS-PRD-3.0-beta1.docx
Normal file
Binary file not shown.
BIN
KS-PRD-3.0-beta2.docx
Normal file
BIN
KS-PRD-3.0-beta2.docx
Normal file
Binary file not shown.
BIN
KS-PRD-3.1-ZK.docx
Normal file
BIN
KS-PRD-3.1-ZK.docx
Normal file
Binary file not shown.
BIN
KS-PRD-3.2-Connect.docx
Normal file
BIN
KS-PRD-3.2-Connect.docx
Normal file
Binary file not shown.
BIN
KS-PRD-3.3-MM2.docx
Normal file
BIN
KS-PRD-3.3-MM2.docx
Normal file
Binary file not shown.
161
README.md
161
README.md
@@ -1,161 +0,0 @@
|
|||||||
|
|
||||||
<p align="center">
|
|
||||||
<img src="https://user-images.githubusercontent.com/71620349/185368586-aed82d30-1534-453d-86ff-ecfa9d0f35bd.png" width = "256" div align=center />
|
|
||||||
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<p align="center">
|
|
||||||
<a href="https://knowstreaming.com">产品官网</a> |
|
|
||||||
<a href="https://github.com/didi/KnowStreaming/releases">下载地址</a> |
|
|
||||||
<a href="https://doc.knowstreaming.com/product">文档资源</a> |
|
|
||||||
<a href="https://demo.knowstreaming.com">体验环境</a>
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<p align="center">
|
|
||||||
<!--最近一次提交时间-->
|
|
||||||
<a href="https://img.shields.io/github/last-commit/didi/KnowStreaming">
|
|
||||||
<img src="https://img.shields.io/github/last-commit/didi/KnowStreaming" alt="LastCommit">
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<!--最新版本-->
|
|
||||||
<a href="https://github.com/didi/KnowStreaming/blob/master/LICENSE">
|
|
||||||
<img src="https://img.shields.io/github/v/release/didi/KnowStreaming" alt="License">
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<!--License信息-->
|
|
||||||
<a href="https://github.com/didi/KnowStreaming/blob/master/LICENSE">
|
|
||||||
<img src="https://img.shields.io/github/license/didi/KnowStreaming" alt="License">
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<!--Open-Issue-->
|
|
||||||
<a href="https://github.com/didi/KnowStreaming/issues">
|
|
||||||
<img src="https://img.shields.io/github/issues-raw/didi/KnowStreaming" alt="Issues">
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<!--知识星球-->
|
|
||||||
<a href="https://z.didi.cn/5gSF9">
|
|
||||||
<img src="https://img.shields.io/badge/join-%E7%9F%A5%E8%AF%86%E6%98%9F%E7%90%83-red" alt="Slack">
|
|
||||||
</a>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## `Know Streaming` 简介
|
|
||||||
|
|
||||||
`Know Streaming`是一套云原生的Kafka管控平台，脱胎于众多互联网内部多年的Kafka运营实践经验，专注于Kafka运维管控、监控告警、资源治理、多活容灾等核心场景。在用户体验、监控、运维管控上进行了平台化、可视化、智能化的建设，提供一系列特色的功能，极大地方便了用户和运维人员的日常使用，让普通运维人员都能成为Kafka专家。
|
|
||||||
|
|
||||||
我们现在正在收集 Know Streaming 用户信息，以帮助我们进一步改进 Know Streaming。
|
|
||||||
请在 [issue#663](https://github.com/didi/KnowStreaming/issues/663) 上提供您的使用信息来支持我们：[谁在使用 Know Streaming](https://github.com/didi/KnowStreaming/issues/663)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
整体具有以下特点：
|
|
||||||
|
|
||||||
- 👀 **零侵入、全覆盖**
|
|
||||||
- 无需侵入改造 `Apache Kafka` ，一键便能纳管 `0.10.x` ~ `3.x.x` 众多版本的Kafka，包括 `ZK` 或 `Raft` 运行模式的版本，同时在兼容架构上具备良好的扩展性，帮助您提升集群管理水平；
|
|
||||||
|
|
||||||
- 🌪️ **零成本、界面化**
|
|
||||||
- 提炼高频 CLI 能力，设计合理的产品路径，提供清新美观的 GUI 界面，支持 Cluster、Broker、Zookeeper、Topic、ConsumerGroup、Message、ACL、Connect 等组件 GUI 管理，普通用户5分钟即可上手；
|
|
||||||
|
|
||||||
- 👏 **云原生、插件化**
|
|
||||||
- 基于云原生构建，具备水平扩展能力，只需要增加节点即可获取更强的采集及对外服务能力，提供众多可热插拔的企业级特性，覆盖可观测性生态整合、资源治理、多活容灾等核心场景；
|
|
||||||
|
|
||||||
- 🚀 **专业能力**
|
|
||||||
- 集群管理：支持一键纳管，健康分析、核心组件观测 等功能；
|
|
||||||
- 观测提升：多维度指标观测大盘、观测指标最佳实践 等功能；
|
|
||||||
- 异常巡检：集群多维度健康巡检、集群多维度健康分 等功能；
|
|
||||||
- 能力增强：集群负载均衡、Topic扩缩副本、Topic副本迁移 等功能；
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**产品图**
|
|
||||||
|
|
||||||
<p align="center">
|
|
||||||
|
|
||||||
<img src="http://img-ys011.didistatic.com/static/dc2img/do1_sPmS4SNLX9m1zlpmHaLJ" width = "768" height = "473" div align=center />
|
|
||||||
|
|
||||||
</p>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 文档资源
|
|
||||||
|
|
||||||
**`开发相关手册`**
|
|
||||||
|
|
||||||
- [打包编译手册](docs/install_guide/源码编译打包手册.md)
|
|
||||||
- [单机部署手册](docs/install_guide/单机部署手册.md)
|
|
||||||
- [版本升级手册](docs/install_guide/版本升级手册.md)
|
|
||||||
- [本地源码启动手册](docs/dev_guide/本地源码启动手册.md)
|
|
||||||
- [页面无数据排查手册](docs/dev_guide/页面无数据排查手册.md)
|
|
||||||
|
|
||||||
**`产品相关手册`**
|
|
||||||
|
|
||||||
- [产品使用指南](docs/user_guide/用户使用手册.md)
|
|
||||||
- [2.x与3.x新旧对比手册](docs/user_guide/新旧对比手册.md)
|
|
||||||
- [FAQ](docs/user_guide/faq.md)
|
|
||||||
|
|
||||||
|
|
||||||
**点击 [这里](https://doc.knowstreaming.com/product)，也可以从官网获取到更多文档**
|
|
||||||
|
|
||||||
**`产品网址`**
|
|
||||||
- [产品官网：https://knowstreaming.com](https://knowstreaming.com)
|
|
||||||
- [体验环境：https://demo.knowstreaming.com](https://demo.knowstreaming.com),登陆账号：admin/admin
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 成为社区贡献者
|
|
||||||
|
|
||||||
1. [贡献源码](https://doc.knowstreaming.com/product/10-contribution) 了解如何成为 Know Streaming 的贡献者
|
|
||||||
2. [具体贡献流程](https://doc.knowstreaming.com/product/10-contribution#102-贡献流程)
|
|
||||||
3. [开源激励计划](https://doc.knowstreaming.com/product/10-contribution#105-开源激励计划)
|
|
||||||
4. [贡献者名单](https://doc.knowstreaming.com/product/10-contribution#106-贡献者名单)
|
|
||||||
|
|
||||||
|
|
||||||
获取KnowStreaming开源社区证书。
|
|
||||||
|
|
||||||
## 加入技术交流群
|
|
||||||
|
|
||||||
**`1、知识星球`**
|
|
||||||
|
|
||||||
<p align="left">
|
|
||||||
<img src="https://user-images.githubusercontent.com/71620349/185357284-fdff1dad-c5e9-4ddf-9a82-0be1c970980d.JPG" height = "180" div align=left />
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
<br/>
|
|
||||||
|
|
||||||
👍 我们正在组建国内最大，最权威的 **[Kafka中文社区](https://z.didi.cn/5gSF9)**
|
|
||||||
|
|
||||||
在这里你可以结交各大互联网的 Kafka大佬 以及 4000+ Kafka爱好者，一起实现知识共享，实时掌控最新行业资讯，期待 👏 您的加入中～ https://z.didi.cn/5gSF9
|
|
||||||
|
|
||||||
有问必答~！ 互动有礼~！
|
|
||||||
|
|
||||||
PS: 提问请尽量把问题一次性描述清楚，并告知环境信息情况～！如使用版本、操作步骤、报错/警告信息等，方便大V们快速解答～
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**`2、微信群`**
|
|
||||||
|
|
||||||
微信加群：添加`PenceXie` 、`szzdzhp001`的微信号备注KnowStreaming加群。
|
|
||||||
<br/>
|
|
||||||
|
|
||||||
加群之前有劳点一下 star，一个小小的 star 是对KnowStreaming作者们努力建设社区的动力。
|
|
||||||
|
|
||||||
感谢感谢！！！
|
|
||||||
|
|
||||||
<img width="116" alt="wx" src="https://user-images.githubusercontent.com/71620349/192257217-c4ebc16c-3ad9-485d-a914-5911d3a4f46b.png">
|
|
||||||
|
|
||||||
## Star History
|
|
||||||
|
|
||||||
[](https://star-history.com/#didi/KnowStreaming&Date)
|
|
||||||
|
|
||||||
@@ -1,646 +0,0 @@
|
|||||||
|
|
||||||
## v3.4.0
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**问题修复**
|
|
||||||
- [Bugfix]修复 Overview 指标文案错误的错误 ([#1190](https://github.com/didi/KnowStreaming/issues/1190))
|
|
||||||
- [Bugfix]修复删除 Kafka 集群后，Connect 集群任务出现 NPE 问题 ([#1129](https://github.com/didi/KnowStreaming/issues/1129))
|
|
||||||
- [Bugfix]修复在 Ldap 登录时，设置 auth-user-registration: false 会导致空指针的问题 ([#1117](https://github.com/didi/KnowStreaming/issues/1117))
|
|
||||||
- [Bugfix]修复 Ldap 登录，调用 user.getId() 出现 NPE 的问题 ([#1108](https://github.com/didi/KnowStreaming/issues/1108))
|
|
||||||
- [Bugfix]修复前端新增角色失败等问题 ([#1107](https://github.com/didi/KnowStreaming/issues/1107))
|
|
||||||
- [Bugfix]修复 ZK 四字命令解析错误的问题
|
|
||||||
- [Bugfix]修复 zk standalone 模式下，状态获取错误的问题
|
|
||||||
- [Bugfix]修复 Broker 元信息解析方法未调用导致接入集群失败的问题 ([#993](https://github.com/didi/KnowStreaming/issues/993))
|
|
||||||
- [Bugfix]修复 ConsumerAssignment 类型转换错误的问题
|
|
||||||
- [Bugfix]修复对 Connect 集群的 clusterUrl 的动态更新导致配置不生效的问题 ([#1079](https://github.com/didi/KnowStreaming/issues/1079))
|
|
||||||
- [Bugfix]修复消费组不支持重置到最旧 Offset 的问题 ([#1059](https://github.com/didi/KnowStreaming/issues/1059))
|
|
||||||
- [Bugfix]后端增加查看 User 密码的权限点 ([#1095](https://github.com/didi/KnowStreaming/issues/1095))
|
|
||||||
- [Bugfix]修复 Connect-JMX 端口维护信息错误的问题 ([#1146](https://github.com/didi/KnowStreaming/issues/1146))
|
|
||||||
- [Bugfix]修复系统管理子应用无法正常启动的问题 ([#1167](https://github.com/didi/KnowStreaming/issues/1167))
|
|
||||||
- [Bugfix]修复 Security 模块，权限点缺失问题 ([#1069](https://github.com/didi/KnowStreaming/issues/1069)), ([#1154](https://github.com/didi/KnowStreaming/issues/1154))
|
|
||||||
- [Bugfix]修复 Connect-Worker Jmx 不生效的问题 ([#1067](https://github.com/didi/KnowStreaming/issues/1067))
|
|
||||||
- [Bugfix]修复权限 ACL 管理中，消费组列表展示错误的问题 ([#1037](https://github.com/didi/KnowStreaming/issues/1037))
|
|
||||||
- [Bugfix]修复 Connect 模块没有默认勾选指标的问题（[#1022](https://github.com/didi/KnowStreaming/issues/1022)）
|
|
||||||
- [Bugfix]修复 es 索引 create/delete 死循环的问题 ([#1021](https://github.com/didi/KnowStreaming/issues/1021))
|
|
||||||
- [Bugfix]修复 Connect-GroupDescription 解析失败的问题 ([#1015](https://github.com/didi/KnowStreaming/issues/1015))
|
|
||||||
- [Bugfix]修复 Prometheus 开放接口中，Partition 指标 tag 缺失的问题 ([#1014](https://github.com/didi/KnowStreaming/issues/1014))
|
|
||||||
- [Bugfix]修复 Topic 消息展示，offset 为 0 不显示的问题 ([#1192](https://github.com/didi/KnowStreaming/issues/1192))
|
|
||||||
- [Bugfix]修复重置offset接口调用过多问题
|
|
||||||
- [Bugfix]Connect 提交任务变更为只保存用户修改的配置，并修复 JSON 模式下配置展示不全的问题 ([#1158](https://github.com/didi/KnowStreaming/issues/1158))
|
|
||||||
- [Bugfix]修复消费组 Offset 重置后，提示重置成功，但是前端不刷新数据，Offset 无变化的问题 ([#1090](https://github.com/didi/KnowStreaming/issues/1090))
|
|
||||||
- [Bugfix]修复未勾选系统管理查看权限，但是依然可以查看系统管理的问题 ([#1105](https://github.com/didi/KnowStreaming/issues/1105))
|
|
||||||
|
|
||||||
|
|
||||||
**产品优化**
|
|
||||||
- [Optimize]补充接入集群时，可选的 Kafka 版本列表 ([#1204](https://github.com/didi/KnowStreaming/issues/1204))
|
|
||||||
- [Optimize]GroupTopic 信息修改为实时获取 ([#1196](https://github.com/didi/KnowStreaming/issues/1196))
|
|
||||||
- [Optimize]增加 AdminClient 观测信息 ([#1111](https://github.com/didi/KnowStreaming/issues/1111))
|
|
||||||
- [Optimize]增加 Connector 运行状态指标 ([#1110](https://github.com/didi/KnowStreaming/issues/1110))
|
|
||||||
- [Optimize]统一 DB 元信息更新格式 ([#1127](https://github.com/didi/KnowStreaming/issues/1127)), ([#1125](https://github.com/didi/KnowStreaming/issues/1125)), ([#1006](https://github.com/didi/KnowStreaming/issues/1006))
|
|
||||||
- [Optimize]日志输出增加支持 MDC，方便用户在 logback.xml 中 json 格式化日志 ([#1032](https://github.com/didi/KnowStreaming/issues/1032))
|
|
||||||
- [Optimize]Jmx 相关日志优化 ([#1082](https://github.com/didi/KnowStreaming/issues/1082))
|
|
||||||
- [Optimize]Topic-Partitions增加主动超时功能 ([#1076](https://github.com/didi/KnowStreaming/issues/1076))
|
|
||||||
- [Optimize]Topic-Messages页面后端增加按照Partition和Offset纬度的排序 ([#1075](https://github.com/didi/KnowStreaming/issues/1075))
|
|
||||||
- [Optimize]Connect-JSON模式下的JSON格式和官方API的格式不一致 ([#1080](https://github.com/didi/KnowStreaming/issues/1080)), ([#1153](https://github.com/didi/KnowStreaming/issues/1153)), ([#1192](https://github.com/didi/KnowStreaming/issues/1192))
|
|
||||||
- [Optimize]登录页面展示的 star 数量修改为最新的数量
|
|
||||||
- [Optimize]Group 列表的 maxLag 指标调整为实时获取 ([#1074](https://github.com/didi/KnowStreaming/issues/1074))
|
|
||||||
- [Optimize]Connector增加重启、编辑、删除等权限点 ([#1066](https://github.com/didi/KnowStreaming/issues/1066)), ([#1147](https://github.com/didi/KnowStreaming/issues/1147))
|
|
||||||
- [Optimize]优化 pom.xml 中，KS版本的标签名
|
|
||||||
- [Optimize]优化集群Brokers中, Controller显示存在延迟的问题 ([#1162](https://github.com/didi/KnowStreaming/issues/1162))
|
|
||||||
- [Optimize]bump jackson version to 2.13.5
|
|
||||||
- [Optimize]权限新增 ACL，自定义权限配置，资源 TransactionalId 优化 ([#1192](https://github.com/didi/KnowStreaming/issues/1192))
|
|
||||||
- [Optimize]Connect 样式优化
|
|
||||||
- [Optimize]消费组详情控制数据实时刷新
|
|
||||||
|
|
||||||
|
|
||||||
**功能新增**
|
|
||||||
- [Feature]新增删除 Group 或 GroupOffset 功能 ([#1064](https://github.com/didi/KnowStreaming/issues/1064)), ([#1084](https://github.com/didi/KnowStreaming/issues/1084)), ([#1040](https://github.com/didi/KnowStreaming/issues/1040)), ([#1144](https://github.com/didi/KnowStreaming/issues/1144))
|
|
||||||
- [Feature]增加 Truncate 数据功能 ([#1062](https://github.com/didi/KnowStreaming/issues/1062)), ([#1043](https://github.com/didi/KnowStreaming/issues/1043)), ([#1145](https://github.com/didi/KnowStreaming/issues/1145))
|
|
||||||
- [Feature]支持指定 Server 的具体 Jmx 端口 ([#965](https://github.com/didi/KnowStreaming/issues/965))
|
|
||||||
|
|
||||||
|
|
||||||
**文档更新**
|
|
||||||
- [Doc]FAQ 补充 ES 8.x 版本使用说明 ([#1189](https://github.com/didi/KnowStreaming/issues/1189))
|
|
||||||
- [Doc]补充启动失败的说明 ([#1126](https://github.com/didi/KnowStreaming/issues/1126))
|
|
||||||
- [Doc]补充 ZK 无数据排查说明 ([#1004](https://github.com/didi/KnowStreaming/issues/1004))
|
|
||||||
- [Doc]无数据排查文档，补充 ES 集群 Shard 满的异常日志
|
|
||||||
- [Doc]README 补充页面无数据排查手册链接
|
|
||||||
- [Doc]补充连接特定 Jmx 端口的说明 ([#965](https://github.com/didi/KnowStreaming/issues/965))
|
|
||||||
- [Doc]补充 zk_properties 字段的使用说明 ([#1003](https://github.com/didi/KnowStreaming/issues/1003))
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## v3.3.0
|
|
||||||
|
|
||||||
**问题修复**
|
|
||||||
- 修复 Connect 的 JMX-Port 配置未生效问题;
|
|
||||||
- 修复 不存在 Connector 时，OverView 页面的数据一直处于加载中的问题;
|
|
||||||
- 修复 Group 分区信息，分页时展示不全的问题；
|
|
||||||
- 修复采集副本指标时，参数传递错误的问题；
|
|
||||||
- 修复用户信息修改后，用户列表会抛出空指针异常的问题；
|
|
||||||
- 修复 Topic 详情页面，查看消息时，选择分区不生效问题；
|
|
||||||
- 修复对 ZK 客户端进行配置后不生效的问题；
|
|
||||||
- 修复 connect 模块，指标中缺少健康巡检项通过数的问题；
|
|
||||||
- 修复 connect 模块，指标获取方法存在映射错误的问题；
|
|
||||||
- 修复 connect 模块，max 纬度指标获取错误的问题；
|
|
||||||
- 修复 Topic 指标大盘 TopN 指标显示信息错误的问题；
|
|
||||||
- 修复 Broker Similar Config 显示错误的问题；
|
|
||||||
- 修复解析 ZK 四字命令时，数据类型设置错误导致空指针的问题；
|
|
||||||
- 修复新增 Topic 时，清理策略选项版本控制错误的问题；
|
|
||||||
- 修复新接入集群时 Controller-Host 信息不显示的问题；
|
|
||||||
- 修复 Connector 和 MM2 列表搜索不生效的问题；
|
|
||||||
- 修复 Zookeeper 页面，Leader 显示存在异常的问题；
|
|
||||||
- 修复前端打包失败的问题；
|
|
||||||
|
|
||||||
|
|
||||||
**产品优化**
|
|
||||||
- ZK Overview 页面补充默认展示的指标；
|
|
||||||
- 统一初始化 ES 索引模版的脚本为 init_es_template.sh，同时新增缺失的 connect 索引模版初始化脚本，去除多余的 replica 和 zookeper 索引模版初始化脚本；
|
|
||||||
- 指标大盘页面，优化指标筛选操作后，无指标数据的指标卡片由不显示改为显示，并增加无数据的兜底；
|
|
||||||
- 删除从 ES 读写 replica 指标的相关代码；
|
|
||||||
- 优化 Topic 健康巡检的日志，明确错误的原因；
|
|
||||||
- 优化无 ZK 模块时，巡检详情忽略对 ZK 的展示；
|
|
||||||
- 优化本地缓存大小为可配置；
|
|
||||||
- Task 模块中的返回中，补充任务的分组信息；
|
|
||||||
- FAQ 补充 Ldap 的配置说明；
|
|
||||||
- FAQ 补充接入 Kerberos 认证的 Kafka 集群的配置说明；
|
|
||||||
- ks_km_kafka_change_record 表增加时间纬度的索引，优化查询性能；
|
|
||||||
- 优化 ZK 健康巡检的日志，便于问题的排查；
|
|
||||||
|
|
||||||
**功能新增**
|
|
||||||
- 新增基于滴滴 Kafka 的 Topic 复制功能（需使用滴滴 Kafka 才可具备该能力）；
|
|
||||||
- Topic 指标大盘，新增 Topic 复制相关的指标；
|
|
||||||
- 新增基于 TestContainers 的单测；
|
|
||||||
|
|
||||||
|
|
||||||
**Kafka MM2 Beta版 (v3.3.0版本新增发布)**
|
|
||||||
- MM2 任务的增删改查；
|
|
||||||
- MM2 任务的指标大盘；
|
|
||||||
- MM2 任务的健康状态；
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## v3.2.0
|
|
||||||
|
|
||||||
**问题修复**
|
|
||||||
- 修复健康巡检结果更新至 DB 时，出现死锁问题；
|
|
||||||
- 修复 KafkaJMXClient 类中，logger错误的问题；
|
|
||||||
- 后端修复 Topic 过期策略在 0.10.1.0 版本能多选的问题，实际应该只能二选一；
|
|
||||||
- 修复接入集群时，不填写集群配置会报错的问题；
|
|
||||||
- 升级 spring-context 至 5.3.19 版本，修复安全漏洞；
|
|
||||||
- 修复 Broker & Topic 修改配置时，多版本兼容配置的版本信息错误的问题；
|
|
||||||
- 修复 Topic 列表的健康分为健康状态；
|
|
||||||
- 修复 Broker LogSize 指标存储名称错误导致查询不到的问题；
|
|
||||||
- 修复 Prometheus 中，缺少 Group 部分指标的问题；
|
|
||||||
- 修复因缺少健康状态指标导致集群数错误的问题；
|
|
||||||
- 修复后台任务记录操作日志时，因缺少操作用户信息导致出现异常的问题；
|
|
||||||
- 修复 Replica 指标查询时，DSL 错误的问题；
|
|
||||||
- 关闭 errorLogger，修复错误日志重复输出的问题；
|
|
||||||
- 修复系统管理更新用户信息失败的问题；
|
|
||||||
- 修复因原AR信息丢失，导致迁移任务一直处于执行中的错误；
|
|
||||||
- 修复集群 Topic 列表实时数据查询时，出现失败的问题；
|
|
||||||
- 修复集群 Topic 列表，页面白屏问题；
|
|
||||||
- 修复副本变更时，因AR数据异常，导致数组访问越界的问题；
|
|
||||||
|
|
||||||
|
|
||||||
**产品优化**
|
|
||||||
- 优化健康巡检为按照资源维度多线程并发处理；
|
|
||||||
- 统一日志输出格式，并优化部分输出的日志；
|
|
||||||
- 优化 ZK 四字命令结果解析过程中，容易引起误解的 WARN 日志；
|
|
||||||
- 优化 Zookeeper 详情中，目录结构的搜索文案；
|
|
||||||
- 优化线程池的名称，方便第三方系统进行相关问题的分析；
|
|
||||||
- 去除 ESClient 的并发访问控制，降低 ESClient 创建数及提升利用率；
|
|
||||||
- 优化 Topic Messages 抽屉文案；
|
|
||||||
- 优化 ZK 健康巡检失败时的错误日志信息；
|
|
||||||
- 提高 Offset 信息获取的超时时间，降低并发过高时出现请求超时的概率；
|
|
||||||
- 优化 Topic & Partition 元信息的更新策略，降低对 DB 连接的占用；
|
|
||||||
- 优化 Sonar 代码扫码问题；
|
|
||||||
- 优化分区 Offset 指标的采集；
|
|
||||||
- 优化前端图表相关组件逻辑;
|
|
||||||
- 优化产品主题色;
|
|
||||||
- Consumer 列表刷新按钮新增 hover 提示；
|
|
||||||
- 优化配置 Topic 的消息大小时的测试弹框体验；
|
|
||||||
- 优化 Overview 页面 TopN 查询的流程；
|
|
||||||
|
|
||||||
|
|
||||||
**功能新增**
|
|
||||||
- 新增页面无数据排查文档；
|
|
||||||
- 增加 ES 索引删除的功能；
|
|
||||||
- 支持拆分API服务和Job服务部署；
|
|
||||||
|
|
||||||
|
|
||||||
**Kafka Connect Beta版 (v3.2.0版本新增发布)**
|
|
||||||
- Connect 集群的纳管；
|
|
||||||
- Connector 的增删改查；
|
|
||||||
- Connect 集群 & Connector 的指标大盘；
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## v3.1.0
|
|
||||||
|
|
||||||
**Bug修复**
|
|
||||||
- 修复重置 Group Offset 的提示信息中，缺少Dead状态也可进行重置的描述；
|
|
||||||
- 修复新建 Topic 后，立即查看 Topic Messages 信息时，会提示 Topic 不存在的问题；
|
|
||||||
- 修复副本变更时，优先副本选举未被正常处罚执行的问题；
|
|
||||||
- 修复 git 目录不存在时，打包不能正常进行的问题；
|
|
||||||
- 修复 KRaft 模式的 Kafka 集群，JMX PORT 显示 -1 的问题；
|
|
||||||
|
|
||||||
|
|
||||||
**体验优化**
|
|
||||||
- 优化Cluster、Broker、Topic、Group的健康分为健康状态；
|
|
||||||
- 去除健康巡检配置中的权重信息；
|
|
||||||
- 错误提示页面展示优化；
|
|
||||||
- 前端打包编译依赖默认使用 taobao 镜像；
|
|
||||||
- 重新设计优化导航栏的 icon ；
|
|
||||||
|
|
||||||
|
|
||||||
**新增**
|
|
||||||
- 个人头像下拉信息中，新增产品版本信息；
|
|
||||||
- 多集群列表页面，新增集群健康状态分布信息；
|
|
||||||
|
|
||||||
|
|
||||||
**Kafka ZK 部分 (v3.1.0版本正式发布)**
|
|
||||||
- 新增 ZK 集群的指标大盘信息；
|
|
||||||
- 新增 ZK 集群的服务状态概览信息；
|
|
||||||
- 新增 ZK 集群的服务节点列表信息；
|
|
||||||
- 新增 Kafka 在 ZK 的存储数据查看功能；
|
|
||||||
- 新增 ZK 的健康巡检及健康状态计算；
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## v3.0.1
|
|
||||||
|
|
||||||
**Bug修复**
|
|
||||||
- 修复重置 Group Offset 时，提示信息中缺少 Dead 状态也可进行重置的信息；
|
|
||||||
- 修复 Ldap 某个属性不存在时，会直接抛出空指针导致登陆失败的问题；
|
|
||||||
- 修复集群 Topic 列表页，健康分详情信息中，检查时间展示错误的问题；
|
|
||||||
- 修复更新健康检查结果时，出现死锁的问题；
|
|
||||||
- 修复 Replica 索引模版错误的问题；
|
|
||||||
- 修复 FAQ 文档中的错误链接；
|
|
||||||
- 修复 Broker 的 TopN 指标不存在时，页面数据不展示的问题；
|
|
||||||
- 修复 Group 详情页，图表时间范围选择不生效的问题；
|
|
||||||
|
|
||||||
|
|
||||||
**体验优化**
|
|
||||||
- 集群 Group 列表按照 Group 维度进行展示；
|
|
||||||
- 优化避免因 ES 中该指标不存在，导致日志中出现大量空指针的问题；
|
|
||||||
- 优化全局 Message & Notification 展示效果；
|
|
||||||
- 优化 Topic 扩分区名称 & 描述展示；
|
|
||||||
|
|
||||||
|
|
||||||
**新增**
|
|
||||||
- Broker 列表页面，新增 JMX 是否成功连接的信息；
|
|
||||||
|
|
||||||
|
|
||||||
**ZK 部分(未完全发布)**
|
|
||||||
- 后端补充 Kafka ZK 指标采集，Kafka ZK 信息获取相关功能；
|
|
||||||
- 增加本地缓存，避免同一采集周期内 ZK 指标重复采集；
|
|
||||||
- 增加 ZK 节点采集失败跳过策略，避免不断对存在问题的节点不断尝试；
|
|
||||||
- 修复 zkAvgLatency 指标转 Long 时抛出异常问题；
|
|
||||||
- 修复 ks_km_zookeeper 表中，role 字段类型错误问题；
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v3.0.0
|
|
||||||
|
|
||||||
**Bug修复**
|
|
||||||
- 修复 Group 指标防重复采集不生效问题
|
|
||||||
- 修复自动创建 ES 索引模版失败问题
|
|
||||||
- 修复 Group+Topic 列表中存在已删除Topic的问题
|
|
||||||
- 修复使用 MySQL-8 ，因兼容问题， start_time 信息为 NULL 时，会导致创建任务失败的问题
|
|
||||||
- 修复 Group 信息表更新时，出现死锁的问题
|
|
||||||
- 修复图表补点逻辑与图表时间范围不适配的问题
|
|
||||||
|
|
||||||
|
|
||||||
**体验优化**
|
|
||||||
- 按照资源类别，拆分健康巡检任务
|
|
||||||
- 优化 Group 详情页的指标为实时获取
|
|
||||||
- 图表拖拽排序支持用户级存储
|
|
||||||
- 多集群列表 ZK 信息展示兼容无 ZK 情况
|
|
||||||
- Topic 详情消息预览支持复制功能
|
|
||||||
- 部分内容大数字支持千位分割符展示
|
|
||||||
|
|
||||||
|
|
||||||
**新增**
|
|
||||||
- 集群信息中，新增 Zookeeper 客户端配置字段
|
|
||||||
- 集群信息中，新增 Kafka 集群运行模式字段
|
|
||||||
- 新增 docker-compose 的部署方式
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v3.0.0-beta.3
|
|
||||||
|
|
||||||
**文档**
|
|
||||||
- FAQ 补充权限识别失败问题的说明
|
|
||||||
- 同步更新文档，保持与官网一致
|
|
||||||
|
|
||||||
|
|
||||||
**Bug修复**
|
|
||||||
- Offset 信息获取时，过滤掉无 Leader 的分区
|
|
||||||
- 升级 oshi-core 版本至 5.6.1 版本，修复 Windows 系统获取系统指标失败问题
|
|
||||||
- 修复 JMX 连接被关闭后，未进行重建的问题
|
|
||||||
- 修复因 DB 中 Broker 信息不存在导致 TotalLogSize 指标获取时抛空指针问题
|
|
||||||
- 修复 dml-logi.sql 中，SQL 注释错误的问题
|
|
||||||
- 修复 startup.sh 中，识别操作系统类型错误的问题
|
|
||||||
- 修复配置管理页面删除配置失败的问题
|
|
||||||
- 修复系统管理应用文件引用路径
|
|
||||||
- 修复 Topic Messages 详情提示信息点击跳转 404 的问题
|
|
||||||
- 修复扩副本时，当前副本数不显示问题
|
|
||||||
|
|
||||||
|
|
||||||
**体验优化**
|
|
||||||
- Topic-Messages 页面，增加返回数据的排序以及按照Earliest/Latest的获取方式
|
|
||||||
- 优化 GroupOffsetResetEnum 类名为 OffsetTypeEnum，使得类名含义更准确
|
|
||||||
- 移动 KafkaZKDAO 类，及 Kafka Znode 实体类的位置，使得 Kafka Zookeeper DAO 更加内聚及便于识别
|
|
||||||
- 后端补充 Overview 页面指标排序的功能
|
|
||||||
- 前端 Webpack 配置优化
|
|
||||||
- Cluster Overview 图表取消放大展示功能
|
|
||||||
- 列表页增加手动刷新功能
|
|
||||||
- 接入/编辑集群，优化 JMX-PORT，Version 信息的回显，优化JMX信息的展示
|
|
||||||
- 提高登录页面图片展示清晰度
|
|
||||||
- 部分样式和文案优化
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v3.0.0-beta.2
|
|
||||||
|
|
||||||
**文档**
|
|
||||||
- 新增登录系统对接文档
|
|
||||||
- 优化前端工程打包构建部分文档说明
|
|
||||||
- FAQ补充KnowStreaming连接特定JMX IP的说明
|
|
||||||
|
|
||||||
|
|
||||||
**Bug修复**
|
|
||||||
- 修复logi_security_oplog表字段过短，导致删除Topic等操作无法记录的问题
|
|
||||||
- 修复ES查询时，抛java.lang.NumberFormatException: For input string: "{"value":0,"relation":"eq"}" 问题
|
|
||||||
- 修复LogStartOffset和LogEndOffset指标单位错误问题
|
|
||||||
- 修复进行副本变更时，旧副本数为NULL的问题
|
|
||||||
- 修复集群Group列表，在第二页搜索时，搜索时返回的分页信息错误问题
|
|
||||||
- 修复重置Offset时，返回的错误信息提示不一致的问题
|
|
||||||
- 修复集群查看，系统查看，LoadRebalance等页面权限点缺失问题
|
|
||||||
- 修复查询不存在的Topic时，错误信息提示不明显的问题
|
|
||||||
- 修复Windows用户打包前端工程报错的问题
|
|
||||||
- package-lock.json锁定前端依赖版本号，修复因依赖自动升级导致打包失败等问题
|
|
||||||
- 系统管理子应用，补充后端返回的Code码拦截，解决后端接口返回报错不展示的问题
|
|
||||||
- 修复用户登出后，依旧可以访问系统的问题
|
|
||||||
- 修复巡检任务配置时，数值显示错误的问题
|
|
||||||
- 修复Broker/Topic Overview 图表和图表详情问题
|
|
||||||
- 修复Job扩缩副本任务明细数据错误的问题
|
|
||||||
- 修复重置Offset时，分区ID，Offset数值无限制问题
|
|
||||||
- 修复扩缩/迁移副本时，无法选中Kafka系统Topic的问题
|
|
||||||
- 修复Topic的Config页面，编辑表单时不能正确回显当前值的问题
|
|
||||||
- 修复Broker Card返回数据后依旧展示加载态的问题
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**体验优化**
|
|
||||||
- 优化默认用户密码为 admin/admin
|
|
||||||
- 缩短新增集群后，集群信息加载的耗时
|
|
||||||
- 集群Broker列表，增加Controller角色信息
|
|
||||||
- 副本变更任务结束后，增加进行优先副本选举的操作
|
|
||||||
- Task模块任务分为Metrics、Common、Metadata三类任务，每类任务配备独立线程池，减少对Job模块的线程池，以及不同类任务之间的相互影响
|
|
||||||
- 删除代码中存在的多余无用文件
|
|
||||||
- 自动新增ES索引模版及近7天索引，减少用户搭建时需要做的事项
|
|
||||||
- 优化前端工程打包流程
|
|
||||||
- 优化登录页文案，页面左侧栏内容，单集群详情样式，Topic列表趋势图等
|
|
||||||
- 首次进入Broker/Topic图表详情时，进行预缓存数据从而优化体验
|
|
||||||
- 优化Topic详情Partition Tab的展示
|
|
||||||
- 多集群列表页增加编辑功能
|
|
||||||
- 优化副本变更时，迁移时间支持分钟级别粒度
|
|
||||||
- logi-security版本升级至2.10.13
|
|
||||||
- logi-elasticsearch-client版本升级至1.0.24
|
|
||||||
|
|
||||||
|
|
||||||
**能力提升**
|
|
||||||
- 支持Ldap登录认证
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v3.0.0-beta.1
|
|
||||||
|
|
||||||
**文档**
|
|
||||||
- 新增Task模块说明文档
|
|
||||||
- FAQ补充 `Specified key was too long; max key length is 767 bytes ` 错误说明
|
|
||||||
- FAQ补充 `出现ESIndexNotFoundException报错` 错误说明
|
|
||||||
|
|
||||||
|
|
||||||
**Bug修复**
|
|
||||||
- 修复 Consumer 点击 Stop 未停止检索的问题
|
|
||||||
- 修复创建/编辑角色权限报错问题
|
|
||||||
- 修复多集群管理/单集群详情均衡卡片状态错误问题
|
|
||||||
- 修复版本列表未排序问题
|
|
||||||
- 修复Raft集群Controller信息不断记录问题
|
|
||||||
- 修复部分版本消费组描述信息获取失败问题
|
|
||||||
- 修复分区Offset获取失败的日志中，缺少Topic名称信息问题
|
|
||||||
- 修复GitHub图地址错误，及图裂问题
|
|
||||||
- 修复Broker默认使用的地址和注释不一致问题
|
|
||||||
- 修复 Consumer 列表分页不生效问题
|
|
||||||
- 修复操作记录表operation_methods字段缺少默认值问题
|
|
||||||
- 修复集群均衡表中move_broker_list字段无效的问题
|
|
||||||
- 修复KafkaUser、KafkaACL信息获取时，日志一直重复提示不支持问题
|
|
||||||
- 修复指标缺失时，曲线出现掉底的问题
|
|
||||||
|
|
||||||
|
|
||||||
**体验优化**
|
|
||||||
- 优化前端构建时间和打包体积，增加依赖打包的分包策略
|
|
||||||
- 优化产品样式和文案展示
|
|
||||||
- 优化ES客户端数为可配置
|
|
||||||
- 优化日志中大量出现的MySQL Key冲突日志
|
|
||||||
|
|
||||||
|
|
||||||
**能力提升**
|
|
||||||
- 增加周期任务，用于主动创建缺少的ES模版及索引的能力，减少额外的脚本操作
|
|
||||||
- 增加JMX连接的Broker地址可选择的能力
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v3.0.0-beta.0
|
|
||||||
|
|
||||||
**1、多集群管理**
|
|
||||||
|
|
||||||
- 增加健康监测体系、关键组件&指标 GUI 展示
|
|
||||||
- 增加 2.8.x 以上 Kafka 集群接入，覆盖 0.10.x-3.x
|
|
||||||
- 删除逻辑集群、共享集群、Region 概念
|
|
||||||
|
|
||||||
**2、Cluster 管理**
|
|
||||||
|
|
||||||
- 增加集群概览信息、集群配置变更记录
|
|
||||||
- 增加 Cluster 健康分，健康检查规则支持自定义配置
|
|
||||||
- 增加 Cluster 关键指标统计和 GUI 展示，支持自定义配置
|
|
||||||
- 增加 Cluster 层 I/O、Disk 的 Load Reblance 功能，支持定时均衡任务（企业版）
|
|
||||||
- 删除限流、鉴权功能
|
|
||||||
- 删除 APPID 概念
|
|
||||||
|
|
||||||
**3、Broker 管理**
|
|
||||||
|
|
||||||
- 增加 Broker 健康分
|
|
||||||
- 增加 Broker 关键指标统计和 GUI 展示，支持自定义配置
|
|
||||||
- 增加 Broker 参数配置功能，需重启生效
|
|
||||||
- 增加 Controller 变更记录
|
|
||||||
- 增加 Broker Datalogs 记录
|
|
||||||
- 删除 Leader Rebalance 功能
|
|
||||||
- 删除 Broker 优先副本选举
|
|
||||||
|
|
||||||
**4、Topic 管理**
|
|
||||||
|
|
||||||
- 增加 Topic 健康分
|
|
||||||
- 增加 Topic 关键指标统计和 GUI 展示，支持自定义配置
|
|
||||||
- 增加 Topic 参数配置功能，可实时生效
|
|
||||||
- 增加 Topic 批量迁移、Topic 批量扩缩副本功能
|
|
||||||
- 增加查看系统 Topic 功能
|
|
||||||
- 优化 Partition 分布的 GUI 展示
|
|
||||||
- 优化 Topic Message 数据采样
|
|
||||||
- 删除 Topic 过期概念
|
|
||||||
- 删除 Topic 申请配额功能
|
|
||||||
|
|
||||||
**5、Consumer 管理**
|
|
||||||
|
|
||||||
- 优化了 ConsumerGroup 展示形式，增加 Consumer Lag 的 GUI 展示
|
|
||||||
|
|
||||||
**6、ACL 管理**
|
|
||||||
|
|
||||||
- 增加原生 ACL GUI 配置功能，可配置生产、消费、自定义多种组合权限
|
|
||||||
- 增加 KafkaUser 功能，可自定义新增 KafkaUser
|
|
||||||
|
|
||||||
**7、消息测试（企业版）**
|
|
||||||
|
|
||||||
- 增加生产者消息模拟器，支持 Data、Flow、Header、Options 自定义配置（企业版）
|
|
||||||
- 增加消费者消息模拟器，支持 Data、Flow、Header、Options 自定义配置（企业版）
|
|
||||||
|
|
||||||
**8、Job**
|
|
||||||
|
|
||||||
- 优化 Job 模块，支持任务进度管理
|
|
||||||
|
|
||||||
**9、系统管理**
|
|
||||||
|
|
||||||
- 优化用户、角色管理体系，支持自定义角色配置页面及操作权限
|
|
||||||
- 优化审计日志信息
|
|
||||||
- 删除多租户体系
|
|
||||||
- 删除工单流程
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## v2.6.0
|
|
||||||
|
|
||||||
版本上线时间：2022-01-24
|
|
||||||
|
|
||||||
### 能力提升
|
|
||||||
- 增加简单回退工具类
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
- 补充周期任务说明文档
|
|
||||||
- 补充集群安装部署使用说明文档
|
|
||||||
- 升级Swagger、SpringFramework、SpringBoot、EChats版本
|
|
||||||
- 优化Task模块的日志输出
|
|
||||||
- 优化corn表达式解析失败后退出无任何日志提示问题
|
|
||||||
- Ldap用户接入时，增加部门及邮箱信息等
|
|
||||||
- 对Jmx模块，增加连接失败后的回退机制及错误日志优化
|
|
||||||
- 增加线程池、客户端池可配置
|
|
||||||
- 删除无用的jmx_prometheus_javaagent-0.14.0.jar
|
|
||||||
- 优化迁移任务名称
|
|
||||||
- 优化创建Region时，Region容量信息不能立即被更新问题
|
|
||||||
- 引入lombok
|
|
||||||
- 更新视频教程
|
|
||||||
- 优化kcm_script.sh脚本中的LogiKM地址为可通过程序传入
|
|
||||||
- 第三方接口及网关接口，增加是否跳过登录的开关
|
|
||||||
- extends模块相关配置调整为非必须在application.yml中配置
|
|
||||||
|
|
||||||
### bug修复
|
|
||||||
- 修复批量往DB写入空指标数组时报SQL语法异常的问题
|
|
||||||
- 修复网关增加配置及修改配置时，version不变化问题
|
|
||||||
- 修复集群列表页，提示框遮挡问题
|
|
||||||
- 修复对高版本Broker元信息协议解析失败的问题
|
|
||||||
- 修复Dockerfile执行时提示缺少application.yml文件的问题
|
|
||||||
- 修复逻辑集群更新时，会报空指针的问题
|
|
||||||
|
|
||||||
|
|
||||||
## v2.5.0
|
|
||||||
|
|
||||||
版本上线时间：2021-07-10
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
- 更改产品名为LogiKM
|
|
||||||
- 更新产品图标
|
|
||||||
|
|
||||||
|
|
||||||
## v2.4.1+
|
|
||||||
|
|
||||||
版本上线时间：2021-05-21
|
|
||||||
|
|
||||||
### 能力提升
|
|
||||||
- 增加直接增加权限和配额的接口(v2.4.1)
|
|
||||||
- 增加接口调用可绕过登录的功能(v2.4.1)
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
- Tomcat 版本提升至8.5.66(v2.4.2)
|
|
||||||
- op接口优化，拆分util接口为topic、leader两类接口(v2.4.1)
|
|
||||||
- 简化Gateway配置的Key长度(v2.4.1)
|
|
||||||
|
|
||||||
### bug修复
|
|
||||||
- 修复页面展示版本错误问题(v2.4.2)
|
|
||||||
|
|
||||||
|
|
||||||
## v2.4.0
|
|
||||||
|
|
||||||
版本上线时间：2021-05-18
|
|
||||||
|
|
||||||
|
|
||||||
### 能力提升
|
|
||||||
|
|
||||||
- 增加App与Topic自动化审批开关
|
|
||||||
- Broker元信息中增加Rack信息
|
|
||||||
- 升级MySQL 驱动，支持MySQL 8+
|
|
||||||
- 增加操作记录查询界面
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
|
|
||||||
- FAQ告警组说明优化
|
|
||||||
- 用户手册共享及 独享集群概念优化
|
|
||||||
- 用户管理界面，前端限制用户删除自己
|
|
||||||
|
|
||||||
### bug修复
|
|
||||||
|
|
||||||
- 修复op-util类中创建Topic失败的接口
|
|
||||||
- 周期同步Topic到DB的任务修复，将Topic列表查询从缓存调整为直接查DB
|
|
||||||
- 应用下线审批失败的功能修复，将权限为0(无权限)的数据进行过滤
|
|
||||||
- 修复登录及权限绕过的漏洞
|
|
||||||
- 修复研发角色展示接入集群、暂停监控等按钮的问题
|
|
||||||
|
|
||||||
|
|
||||||
## v2.3.0
|
|
||||||
|
|
||||||
版本上线时间：2021-02-08
|
|
||||||
|
|
||||||
|
|
||||||
### 能力提升
|
|
||||||
|
|
||||||
- 新增支持docker化部署
|
|
||||||
- 可指定Broker作为候选controller
|
|
||||||
- 可新增并管理网关配置
|
|
||||||
- 可获取消费组状态
|
|
||||||
- 增加集群的JMX认证
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
|
|
||||||
- 优化编辑用户角色、修改密码的流程
|
|
||||||
- 新增consumerID的搜索功能
|
|
||||||
- 优化“Topic连接信息”、“消费组重置消费偏移”、“修改Topic保存时间”的文案提示
|
|
||||||
- 在相应位置增加《资源申请文档》链接
|
|
||||||
|
|
||||||
### bug修复
|
|
||||||
|
|
||||||
- 修复Broker监控图表时间轴展示错误的问题
|
|
||||||
- 修复创建夜莺监控告警规则时，使用的告警周期的单位不正确的问题
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## v2.2.0
|
|
||||||
|
|
||||||
版本上线时间：2021-01-25
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 能力提升
|
|
||||||
|
|
||||||
- 优化工单批量操作流程
|
|
||||||
- 增加获取Topic75分位/99分位的实时耗时数据
|
|
||||||
- 增加定时任务，可将无主未落DB的Topic定期写入DB
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
|
|
||||||
- 在相应位置增加《集群接入文档》链接
|
|
||||||
- 优化物理集群、逻辑集群含义
|
|
||||||
- 在Topic详情页、Topic扩分区操作弹窗增加展示Topic所属Region的信息
|
|
||||||
- 优化Topic审批时，Topic数据保存时间的配置流程
|
|
||||||
- 优化Topic/应用申请、审批时的错误提示文案
|
|
||||||
- 优化Topic数据采样的操作项文案
|
|
||||||
- 优化运维人员删除Topic时的提示文案
|
|
||||||
- 优化运维人员删除Region的删除逻辑与提示文案
|
|
||||||
- 优化运维人员删除逻辑集群的提示文案
|
|
||||||
- 优化上传集群配置文件时的文件类型限制条件
|
|
||||||
|
|
||||||
### bug修复
|
|
||||||
|
|
||||||
- 修复填写应用名称时校验特殊字符出错的问题
|
|
||||||
- 修复普通用户越权访问应用详情的问题
|
|
||||||
- 修复由于Kafka版本升级，导致的数据压缩格式无法获取的问题
|
|
||||||
- 修复删除逻辑集群或Topic之后，界面依旧展示的问题
|
|
||||||
- 修复进行Leader rebalance操作时执行结果重复提示的问题
|
|
||||||
|
|
||||||
|
|
||||||
## v2.1.0
|
|
||||||
|
|
||||||
版本上线时间：2020-12-19
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 体验优化
|
|
||||||
|
|
||||||
- 优化页面加载时的背景样式
|
|
||||||
- 优化普通用户申请Topic权限的流程
|
|
||||||
- 优化Topic申请配额、申请分区的权限限制
|
|
||||||
- 优化取消Topic权限的文案提示
|
|
||||||
- 优化申请配额表单的表单项名称
|
|
||||||
- 优化重置消费偏移的操作流程
|
|
||||||
- 优化创建Topic迁移任务的表单内容
|
|
||||||
- 优化Topic扩分区操作的弹窗界面样式
|
|
||||||
- 优化集群Broker监控可视化图表样式
|
|
||||||
- 优化创建逻辑集群的表单内容
|
|
||||||
- 优化集群安全协议的提示文案
|
|
||||||
|
|
||||||
### bug修复
|
|
||||||
|
|
||||||
- 修复偶发性重置消费偏移失败的问题
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,16 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
cd `dirname $0`/../libs
|
|
||||||
target_dir=`pwd`
|
|
||||||
|
|
||||||
pid=`ps ax | grep -i 'ks-km' | grep ${target_dir} | grep java | grep -v grep | awk '{print $1}'`
|
|
||||||
if [ -z "$pid" ] ; then
|
|
||||||
echo "No ks-km running."
|
|
||||||
exit -1;
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "The ks-km (${pid}) is running..."
|
|
||||||
|
|
||||||
kill ${pid}
|
|
||||||
|
|
||||||
echo "Send shutdown request to ks-km (${pid}) OK"
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
error_exit ()
|
|
||||||
{
|
|
||||||
echo "ERROR: $1 !!"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java
|
|
||||||
[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/usr/java
|
|
||||||
[ ! -e "$JAVA_HOME/bin/java" ] && unset JAVA_HOME
|
|
||||||
|
|
||||||
if [ -z "$JAVA_HOME" ]; then
|
|
||||||
if [ "Darwin" = "$(uname -s)" ]; then
|
|
||||||
|
|
||||||
if [ -x '/usr/libexec/java_home' ] ; then
|
|
||||||
export JAVA_HOME=`/usr/libexec/java_home`
|
|
||||||
|
|
||||||
elif [ -d "/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" ]; then
|
|
||||||
export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
JAVA_PATH=`dirname $(readlink -f $(which javac))`
|
|
||||||
if [ "x$JAVA_PATH" != "x" ]; then
|
|
||||||
export JAVA_HOME=`dirname $JAVA_PATH 2>/dev/null`
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [ -z "$JAVA_HOME" ]; then
|
|
||||||
error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)! jdk8 or later is better!"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
export WEB_SERVER="ks-km"
|
|
||||||
export JAVA_HOME
|
|
||||||
export JAVA="$JAVA_HOME/bin/java"
|
|
||||||
export BASE_DIR=`cd $(dirname $0)/..; pwd`
|
|
||||||
export CUSTOM_SEARCH_LOCATIONS=file:${BASE_DIR}/conf/
|
|
||||||
|
|
||||||
|
|
||||||
#===========================================================================================
|
|
||||||
# JVM Configuration
|
|
||||||
#===========================================================================================
|
|
||||||
|
|
||||||
JAVA_OPT="${JAVA_OPT} -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m"
|
|
||||||
JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof"
|
|
||||||
|
|
||||||
## jdk版本高的情况 有些 参数废弃了
|
|
||||||
JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
|
|
||||||
if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
|
|
||||||
JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/km_gc.log:time,tags:filecount=10,filesize=102400"
|
|
||||||
else
|
|
||||||
JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext"
|
|
||||||
JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/km_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/libs/${WEB_SERVER}.jar"
|
|
||||||
JAVA_OPT="${JAVA_OPT} --spring.config.additional-location=${CUSTOM_SEARCH_LOCATIONS}"
|
|
||||||
JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/logback-spring.xml"
|
|
||||||
JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if [ ! -d "${BASE_DIR}/logs" ]; then
|
|
||||||
mkdir ${BASE_DIR}/logs
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$JAVA ${JAVA_OPT}"
|
|
||||||
|
|
||||||
# check the start.out log output file
|
|
||||||
if [ ! -f "${BASE_DIR}/logs/start.out" ]; then
|
|
||||||
touch "${BASE_DIR}/logs/start.out"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# start
|
|
||||||
echo -e "---- 启动脚本 ------\n $JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 &
|
|
||||||
|
|
||||||
|
|
||||||
nohup $JAVA ${JAVA_OPT} >> ${BASE_DIR}/logs/start.out 2>&1 &
|
|
||||||
|
|
||||||
echo "${WEB_SERVER} is starting，you can check the ${BASE_DIR}/logs/start.out"
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
<mxfile host="65bd71144e">
|
|
||||||
<diagram id="vxzhwhZdNVAY19FZ4dgb" name="Page-1">
|
|
||||||
<mxGraphModel dx="1194" dy="733" grid="0" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
|
|
||||||
<root>
|
|
||||||
<mxCell id="0"/>
|
|
||||||
<mxCell id="1" parent="0"/>
|
|
||||||
<mxCell id="4" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;startArrow=none;strokeWidth=2;strokeColor=#6666FF;" edge="1" parent="1" source="16">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="200" y="540" as="targetPoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="7" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;strokeColor=#33FF33;strokeWidth=2;" edge="1" parent="1" source="2">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="360" y="240" as="targetPoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="5" style="edgeStyle=none;html=1;startArrow=none;strokeColor=#33FF33;strokeWidth=2;" edge="1" parent="1">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="200" y="400" as="targetPoint"/>
|
|
||||||
<mxPoint x="360" y="360" as="sourcePoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="3" value="C3" style="verticalLabelPosition=middle;verticalAlign=middle;html=1;shape=mxgraph.flowchart.on-page_reference;labelPosition=center;align=center;strokeColor=#FF8000;strokeWidth=2;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="340" y="280" width="40" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="18" style="edgeStyle=none;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;endArrow=none;endFill=0;strokeColor=#FF8000;strokeWidth=2;" edge="1" parent="1" source="8" target="3">
|
|
||||||
<mxGeometry relative="1" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="8" value="fix_928" style="rounded=1;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="320" y="40" width="80" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="9" value="github_master" style="rounded=1;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="160" y="40" width="80" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="10" value="" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;endArrow=classic;startArrow=none;endFill=1;strokeWidth=2;strokeColor=#6666FF;" edge="1" parent="1" source="11" target="2">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="200" y="640" as="targetPoint"/>
|
|
||||||
<mxPoint x="200" y="80" as="sourcePoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="2" value="C2" style="verticalLabelPosition=middle;verticalAlign=middle;html=1;shape=mxgraph.flowchart.on-page_reference;labelPosition=center;align=center;strokeColor=#6666FF;strokeWidth=2;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="180" y="200" width="40" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="12" value="" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;endArrow=classic;endFill=1;strokeWidth=2;strokeColor=#6666FF;" edge="1" parent="1" source="9" target="11">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="200" y="200" as="targetPoint"/>
|
|
||||||
<mxPoint x="200" y="80" as="sourcePoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="11" value="C1" style="verticalLabelPosition=middle;verticalAlign=middle;html=1;shape=mxgraph.flowchart.on-page_reference;labelPosition=center;align=center;strokeColor=#6666FF;strokeWidth=2;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="180" y="120" width="40" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="23" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;exitPerimeter=0;endArrow=none;endFill=0;strokeColor=#FF8000;strokeWidth=2;" edge="1" parent="1" source="3">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="360" y="360" as="targetPoint"/>
|
|
||||||
<mxPoint x="360" y="400" as="sourcePoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="17" value="" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;startArrow=none;endArrow=none;strokeWidth=2;strokeColor=#6666FF;" edge="1" parent="1" source="2" target="16">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="200" y="640" as="targetPoint"/>
|
|
||||||
<mxPoint x="200" y="240" as="sourcePoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="16" value="C4" style="verticalLabelPosition=middle;verticalAlign=middle;html=1;shape=mxgraph.flowchart.on-page_reference;labelPosition=center;align=center;strokeColor=#6666FF;strokeWidth=2;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="180" y="440" width="40" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="22" value="Tag-v3.2.0" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;fillColor=none;strokeColor=none;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="100" y="120" width="80" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="24" value="Tag-v3.2.1" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;fillColor=none;strokeColor=none;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="100" y="440" width="80" height="40" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="27" value="切换到主分支：git checkout github_master" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;labelPosition=center;verticalLabelPosition=middle;align=center;verticalAlign=middle;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="520" y="90" width="240" height="30" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="34" style="edgeStyle=none;html=1;exitX=0;exitY=0;exitDx=0;exitDy=0;entryX=0.855;entryY=0.145;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;fontSize=18;endArrow=none;endFill=0;" edge="1" parent="1" source="28" target="2">
|
|
||||||
<mxGeometry relative="1" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="28" value="主分支拉最新代码：git pull" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;labelPosition=center;verticalLabelPosition=middle;align=center;verticalAlign=middle;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="520" y="120" width="160" height="30" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="35" style="edgeStyle=none;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;dashed=1;dashPattern=8 8;fontSize=18;endArrow=none;endFill=0;" edge="1" parent="1" source="29">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="270" y="225" as="targetPoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="29" value="基于主分支拉新分支：git checkout -b fix_928" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;labelPosition=center;verticalLabelPosition=middle;align=center;verticalAlign=middle;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="520" y="210" width="250" height="30" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="37" style="edgeStyle=none;html=1;exitX=0;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;fontSize=18;endArrow=none;endFill=0;" edge="1" parent="1" source="30" target="3">
|
|
||||||
<mxGeometry relative="1" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="30" value="提交代码：git commit -m "[Optimize]优化xxx问题(#928)"" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;labelPosition=center;verticalLabelPosition=middle;align=center;verticalAlign=middle;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="520" y="270" width="320" height="30" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="31" value="提交到自己远端仓库：git push --set-upstream origin fix_928" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;labelPosition=center;verticalLabelPosition=middle;align=center;verticalAlign=middle;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="520" y="300" width="334" height="30" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="38" style="edgeStyle=none;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;dashed=1;dashPattern=8 8;fontSize=18;endArrow=none;endFill=0;" edge="1" parent="1" source="32">
|
|
||||||
<mxGeometry relative="1" as="geometry">
|
|
||||||
<mxPoint x="280" y="380" as="targetPoint"/>
|
|
||||||
</mxGeometry>
|
|
||||||
</mxCell>
|
|
||||||
<mxCell id="32" value="GitHub页面发起Pull Request请求，管理员合入主仓库" style="rounded=0;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=0;labelPosition=center;verticalLabelPosition=middle;align=center;verticalAlign=middle;" vertex="1" parent="1">
|
|
||||||
<mxGeometry x="520" y="360" width="300" height="30" as="geometry"/>
|
|
||||||
</mxCell>
|
|
||||||
</root>
|
|
||||||
</mxGraphModel>
|
|
||||||
</diagram>
|
|
||||||
</mxfile>
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 180 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 80 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 631 KiB |
@@ -1 +0,0 @@
|
|||||||
TODO.
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
# 贡献名单
|
|
||||||
|
|
||||||
- [贡献名单](#贡献名单)
|
|
||||||
- [1、贡献者角色](#1贡献者角色)
|
|
||||||
- [1.1、Maintainer](#11maintainer)
|
|
||||||
- [1.2、Committer](#12committer)
|
|
||||||
- [1.3、Contributor](#13contributor)
|
|
||||||
- [2、贡献者名单](#2贡献者名单)
|
|
||||||
|
|
||||||
|
|
||||||
## 1、贡献者角色
|
|
||||||
|
|
||||||
KnowStreaming 开发者包含 Maintainer、Committer、Contributor 三种角色，每种角色的标准定义如下。
|
|
||||||
|
|
||||||
### 1.1、Maintainer
|
|
||||||
|
|
||||||
Maintainer 是对 KnowStreaming 项目的演进和发展做出显著贡献的个人。具体包含以下的标准：
|
|
||||||
|
|
||||||
- 完成多个关键模块或者工程的设计与开发，是项目的核心开发人员；
|
|
||||||
- 持续的投入和激情，能够积极参与社区、官网、issue、PR 等项目相关事项的维护；
|
|
||||||
- 在社区中具有有目共睹的影响力，能够代表 KnowStreaming 参加重要的社区会议和活动；
|
|
||||||
- 具有培养 Committer 和 Contributor 的意识和能力；
|
|
||||||
|
|
||||||
### 1.2、Committer
|
|
||||||
|
|
||||||
Committer 是具有 KnowStreaming 仓库写权限的个人，包含以下的标准：
|
|
||||||
|
|
||||||
- 能够在长时间内做持续贡献 issue、PR 的个人；
|
|
||||||
- 参与 issue 列表的维护及重要 feature 的讨论；
|
|
||||||
- 参与 code review；
|
|
||||||
|
|
||||||
### 1.3、Contributor
|
|
||||||
|
|
||||||
Contributor 是对 KnowStreaming 项目有贡献的个人，标准为：
|
|
||||||
|
|
||||||
- 提交过 PR 并被合并；
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2、贡献者名单
|
|
||||||
|
|
||||||
开源贡献者名单(不定期更新)
|
|
||||||
|
|
||||||
在名单内,但是没有收到贡献者礼品的同学,可以联系：szzdzhp001
|
|
||||||
|
|
||||||
| 姓名 | Github | 角色 | 公司 |
|
|
||||||
| ------------------- | ---------------------------------------------------------- | ----------- | -------- |
|
|
||||||
| 张亮 | [@zhangliangboy](https://github.com/zhangliangboy) | Maintainer | 滴滴出行 |
|
|
||||||
| 谢鹏 | [@PenceXie](https://github.com/PenceXie) | Maintainer | 滴滴出行 |
|
|
||||||
| 赵情融 | [@zqrferrari](https://github.com/zqrferrari) | Maintainer | 滴滴出行 |
|
|
||||||
| 石臻臻 | [@shirenchuang](https://github.com/shirenchuang) | Maintainer | 滴滴出行 |
|
|
||||||
| 曾巧 | [@ZQKC](https://github.com/ZQKC) | Maintainer | 滴滴出行 |
|
|
||||||
| 孙超 | [@lucasun](https://github.com/lucasun) | Maintainer | 滴滴出行 |
|
|
||||||
| 洪华驰 | [@brodiehong](https://github.com/brodiehong) | Maintainer | 滴滴出行 |
|
|
||||||
| 许喆 | [@potaaaaaato](https://github.com/potaaaaaato) | Committer | 滴滴出行 |
|
|
||||||
| 郭宇航 | [@GraceWalk](https://github.com/GraceWalk) | Committer | 滴滴出行 |
|
|
||||||
| 李伟 | [@velee](https://github.com/velee) | Committer | 滴滴出行 |
|
|
||||||
| 张占昌 | [@zzccctv](https://github.com/zzccctv) | Committer | 滴滴出行 |
|
|
||||||
| 王东方 | [@wangdongfang-aden](https://github.com/wangdongfang-aden) | Committer | 滴滴出行 |
|
|
||||||
| 王耀波 | [@WYAOBO](https://github.com/WYAOBO) | Committer | 滴滴出行 |
|
|
||||||
| 赵寅锐 | [@ZHAOYINRUI](https://github.com/ZHAOYINRUI) | Maintainer | 字节跳动 |
|
|
||||||
| haoqi123 | [@haoqi123](https://github.com/haoqi123) | Contributor | 前程无忧 |
|
|
||||||
| chaixiaoxue | [@chaixiaoxue](https://github.com/chaixiaoxue) | Contributor | SYNNEX |
|
|
||||||
| 陆晗 | [@luhea](https://github.com/luhea) | Contributor | 竞技世界 |
|
|
||||||
| Mengqi777 | [@Mengqi777](https://github.com/Mengqi777) | Contributor | 腾讯 |
|
|
||||||
| ruanliang-hualun | [@ruanliang-hualun](https://github.com/ruanliang-hualun) | Contributor | 网易 |
|
|
||||||
| 17hao | [@17hao](https://github.com/17hao) | Contributor | |
|
|
||||||
| Huyueeer | [@Huyueeer](https://github.com/Huyueeer) | Contributor | INVENTEC |
|
|
||||||
| lomodays207 | [@lomodays207](https://github.com/lomodays207) | Contributor | 建信金科 |
|
|
||||||
| Super .Wein（星痕） | [@superspeedone](https://github.com/superspeedone) | Contributor | 韵达 |
|
|
||||||
| Hongten | [@Hongten](https://github.com/Hongten) | Contributor | Shopee |
|
|
||||||
| 徐正熙 | [@hyper-xx)](https://github.com/hyper-xx) | Contributor | 滴滴出行 |
|
|
||||||
| RichardZhengkay | [@RichardZhengkay](https://github.com/RichardZhengkay) | Contributor | 趣街 |
|
|
||||||
| 罐子里的茶 | [@gzldc](https://github.com/gzldc) | Contributor | 道富 |
|
|
||||||
| 陈忠玉 | [@paula](https://github.com/chenzhongyu11) | Contributor | 平安产险 |
|
|
||||||
| 杨光 | [@yaangvipguang](https://github.com/yangvipguang) | Contributor |
|
|
||||||
| 王亚聪 | [@wangyacongi](https://github.com/wangyacongi) | Contributor |
|
|
||||||
| Yang Jing | [@yangbajing](https://github.com/yangbajing) | Contributor | |
|
|
||||||
| 刘新元 Liu XinYuan | [@Liu-XinYuan](https://github.com/Liu-XinYuan) | Contributor | |
|
|
||||||
| Joker | [@LiubeyJokerQueue](https://github.com/JokerQueue) | Contributor | 丰巢 |
|
|
||||||
| Eason Lau | [@Liubey](https://github.com/Liubey) | Contributor | |
|
|
||||||
| hailanxin | [@hailanxin](https://github.com/hailanxin) | Contributor | |
|
|
||||||
| Qi Zhang | [@zzzhangqi](https://github.com/zzzhangqi) | Contributor | 好雨科技 |
|
|
||||||
| fengxsong | [@fengxsong](https://github.com/fengxsong) | Contributor | |
|
|
||||||
| 谢晓东 | [@Strangevy](https://github.com/Strangevy) | Contributor | 花生日记 |
|
|
||||||
| ZhaoXinlong | [@ZhaoXinlong](https://github.com/ZhaoXinlong) | Contributor | |
|
|
||||||
| xuehaipeng | [@xuehaipeng](https://github.com/xuehaipeng) | Contributor | |
|
|
||||||
| 孔令续 | [@mrazkong](https://github.com/mrazkong) | Contributor | |
|
|
||||||
| pierre xiong | [@pierre94](https://github.com/pierre94) | Contributor | |
|
|
||||||
| PengShuaixin | [@PengShuaixin](https://github.com/PengShuaixin) | Contributor | |
|
|
||||||
| 梁壮 | [@lz](https://github.com/silent-night-no-trace) | Contributor | |
|
|
||||||
| 张晓寅 | [@ahu0605](https://github.com/ahu0605) | Contributor | 电信数智 |
|
|
||||||
| 黄海婷 | [@Huanghaiting](https://github.com/Huanghaiting) | Contributor | 云徙科技 |
|
|
||||||
| 任祥德 | [@RenChauncy](https://github.com/RenChauncy) | Contributor | 探马企服 |
|
|
||||||
| 胡圣林 | [@slhu997](https://github.com/slhu997) | Contributor | |
|
|
||||||
| 史泽颖 | [@shizeying](https://github.com/shizeying) | Contributor | |
|
|
||||||
| 王玉博 | [@Wyb7290](https://github.com/Wyb7290) | Committer | |
|
|
||||||
| 伍璇 | [@Luckywustone](https://github.com/Luckywustone) | Contributor ||
|
|
||||||
| 邓苑 | [@CatherineDY](https://github.com/CatherineDY) | Contributor ||
|
|
||||||
| 封琼凤 | [@Luckywustone](https://github.com/fengqiongfeng) | Committer ||
|
|
||||||
@@ -1,168 +0,0 @@
|
|||||||
# 贡献指南
|
|
||||||
|
|
||||||
- [贡献指南](#贡献指南)
|
|
||||||
- [1、行为准则](#1行为准则)
|
|
||||||
- [2、仓库规范](#2仓库规范)
|
|
||||||
- [2.1、Issue 规范](#21issue-规范)
|
|
||||||
- [2.2、Commit-Log 规范](#22commit-log-规范)
|
|
||||||
- [2.3、Pull-Request 规范](#23pull-request-规范)
|
|
||||||
- [3、操作示例](#3操作示例)
|
|
||||||
- [3.1、初始化环境](#31初始化环境)
|
|
||||||
- [3.2、认领问题](#32认领问题)
|
|
||||||
- [3.3、处理问题 \& 提交解决](#33处理问题--提交解决)
|
|
||||||
- [3.4、请求合并](#34请求合并)
|
|
||||||
- [4、常见问题](#4常见问题)
|
|
||||||
- [4.1、如何将多个 Commit-Log 合并为一个？](#41如何将多个-commit-log-合并为一个)
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
欢迎 👏🏻 👏🏻 👏🏻 来到 `KnowStreaming`。本文档是关于如何为 `KnowStreaming` 做出贡献的指南。如果您发现不正确或遗漏的内容, 请留下您的意见/建议。
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## 1、行为准则
|
|
||||||
|
|
||||||
请务必阅读并遵守我们的：[行为准则](https://github.com/didi/KnowStreaming/blob/master/CODE_OF_CONDUCT.md)。
|
|
||||||
|
|
||||||
|
|
||||||
## 2、仓库规范
|
|
||||||
|
|
||||||
### 2.1、Issue 规范
|
|
||||||
|
|
||||||
按要求，在 [创建Issue](https://github.com/didi/KnowStreaming/issues/new/choose) 中创建ISSUE即可。
|
|
||||||
|
|
||||||
需要重点说明的是：
|
|
||||||
- 提供出现问题的环境信息，包括使用的系统，使用的KS版本等；
|
|
||||||
- 提供出现问题的复现方式；
|
|
||||||
|
|
||||||
|
|
||||||
### 2.2、Commit-Log 规范
|
|
||||||
|
|
||||||
`Commit-Log` 包含三部分 `Header`、`Body`、`Footer`。其中 `Header` 是必须的，格式固定，`Body` 在变更有必要详细解释时使用。
|
|
||||||
|
|
||||||
|
|
||||||
**1、`Header` 规范**
|
|
||||||
|
|
||||||
`Header` 格式为 `[Type]Message`， 主要有三部分组成，分别是`Type`、`Message`，
|
|
||||||
|
|
||||||
- `Type`：说明这个提交是哪一个类型的，比如有 Bugfix、Feature、Optimize等；
|
|
||||||
- `Message`：说明提交的信息，比如修复xx问题；
|
|
||||||
|
|
||||||
|
|
||||||
实际例子：[`[Bugfix]修复新接入的集群，Controller-Host不显示的问题`](https://github.com/didi/KnowStreaming/pull/933/commits)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**2、`Body` 规范**
|
|
||||||
|
|
||||||
一般不需要，如果解决了较复杂问题，或者代码较多，需要 `Body` 说清楚解决的问题，解决的思路等信息。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**3、实际例子**
|
|
||||||
|
|
||||||
```
|
|
||||||
[Optimize]优化 MySQL & ES 测试容器的初始化
|
|
||||||
|
|
||||||
主要的变更
|
|
||||||
1、knowstreaming/knowstreaming-manager 容器；
|
|
||||||
2、knowstreaming/knowstreaming-mysql 容器调整为使用 mysql:5.7 容器；
|
|
||||||
3、初始化 mysql:5.7 容器后，增加初始化 MySQL 表及数据的动作；
|
|
||||||
|
|
||||||
被影响的变更：
|
|
||||||
1、移动 km-dist/init/sql 下的MySQL初始化脚本至 km-persistence/src/main/resource/sql 下，以便项目测试时加载到所需的初始化 SQL；
|
|
||||||
2、删除无用的 km-dist/init/template 目录；
|
|
||||||
3、因为 km-dist/init/sql 和 km-dist/init/template 目录的调整，因此也调整 ReleaseKnowStreaming.xml 内的文件内容；
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
**TODO : 后续有兴趣的同学，可以考虑引入 Git 的 Hook 进行更好的 Commit-Log 的管理。**
|
|
||||||
|
|
||||||
|
|
||||||
### 2.3、Pull-Request 规范
|
|
||||||
|
|
||||||
详细见：[PULL-REQUEST 模版](../../.github/PULL_REQUEST_TEMPLATE.md)
|
|
||||||
|
|
||||||
需要重点说明的是：
|
|
||||||
|
|
||||||
- <font color=red > 任何 PR 都必须与有效 ISSUE 相关联。否则， PR 将被拒绝；</font>
|
|
||||||
- <font color=red> 一个分支只修改一件事，一个 PR 只修改一件事；</b></font>
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## 3、操作示例
|
|
||||||
|
|
||||||
本节主要介绍对 `KnowStreaming` 进行代码贡献时，相关的操作方式及操作命令。
|
|
||||||
|
|
||||||
名词说明：
|
|
||||||
- 主仓库：https://github.com/didi/KnowStreaming 这个仓库为主仓库。
|
|
||||||
- 分仓库：Fork 到自己账号下的 KnowStreaming 仓库为分仓库；
|
|
||||||
|
|
||||||
|
|
||||||
### 3.1、初始化环境
|
|
||||||
|
|
||||||
1. `Fork KnowStreaming` 主仓库至自己账号下，见 https://github.com/didi/KnowStreaming 地址右上角的 `Fork` 按钮；
|
|
||||||
2. 克隆分仓库至本地：`git clone git@github.com:xxxxxxx/KnowStreaming.git`，该仓库的简写名通常是`origin`；
|
|
||||||
3. 添加主仓库至本地：`git remote add upstream https://github.com/didi/KnowStreaming`，`upstream`是主仓库在本地的简写名，可以随意命名，前后保持一致即可；
|
|
||||||
4. 拉取主仓库代码：`git fetch upstream`；
|
|
||||||
5. 拉取分仓库代码：`git fetch origin`；
|
|
||||||
6. 将主仓库的`master`分支，拉取到本地并命名为`github_master`：`git checkout -b upstream/master`；
|
|
||||||
|
|
||||||
最后，我们来看一下初始化完成之后的大致效果，具体如下图所示：
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
至此，我们的环境就初始化好了。后续，`github_master` 分支就是主仓库的`master`分支，我们可以使用`git pull`拉取该分支的最新代码，还可以使用`git checkout -b xxx`拉取我们想要的分支。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 3.2、认领问题
|
|
||||||
|
|
||||||
在文末评论说明自己要处理该问题即可，具体如下图所示：
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 3.3、处理问题 & 提交解决
|
|
||||||
|
|
||||||
本节主要介绍一下处理问题 & 提交解决过程中的分支管理，具体如下图所示：
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
1. 切换到主分支：`git checkout github_master`；
|
|
||||||
2. 主分支拉最新代码：`git pull`；
|
|
||||||
3. 基于主分支拉新分支：`git checkout -b fix_928`；
|
|
||||||
4. 提交代码，安装commit的规范进行提交，例如：`git commit -m "[Optimize]优化xxx问题"`；
|
|
||||||
5. 提交到自己远端仓库：`git push --set-upstream origin fix_928`；
|
|
||||||
6. `GitHub` 页面发起 `Pull Request` 请求，管理员合入主仓库。这部分详细见下一节；
|
|
||||||
|
|
||||||
|
|
||||||
### 3.4、请求合并
|
|
||||||
|
|
||||||
代码在提交到 `GitHub` 分仓库之后，就可以在 `GitHub` 的网站创建 `Pull Request`，申请将代码合入主仓库了。 `Pull Request` 具体见下图所示：
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
[Pull Request 创建的例子](https://github.com/didi/KnowStreaming/pull/945)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## 4、常见问题
|
|
||||||
|
|
||||||
### 4.1、如何将多个 Commit-Log 合并为一个？
|
|
||||||
|
|
||||||
可以不需要将多个commit合并为一个，如果要合并，可以使用 `git rebase -i` 命令进行解决。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,264 +0,0 @@
|
|||||||
# Task模块简介
|
|
||||||
|
|
||||||
## 1、Task简介
|
|
||||||
|
|
||||||
在 KnowStreaming 中(下面简称KS)，Task模块主要是用于执行一些周期任务，包括Cluster、Broker、Topic等指标的定时采集，集群元数据定时更新至DB，集群状态的健康巡检等。在KS中，与Task模块相关的代码，我们都统一存放在km-task模块中。
|
|
||||||
|
|
||||||
Task模块是基于 LogiCommon 中的Logi-Job组件实现的任务周期执行，Logi-Job 的功能类似 XXX-Job，它是 XXX-Job 在 KnowStreaming 的内嵌实现，主要用于简化 KnowStreaming 的部署。
|
|
||||||
Logi-Job 的任务总共有两种执行模式，分别是：
|
|
||||||
|
|
||||||
+ 广播模式：同一KS集群下，同一任务周期中，所有KS主机都会执行该定时任务。
|
|
||||||
+ 抢占模式：同一KS集群下，同一任务周期中，仅有某一台KS主机会执行该任务。
|
|
||||||
|
|
||||||
KS集群范围定义：连接同一个DB，且application.yml中的spring.logi-job.app-name的名称一样的KS主机为同一KS集群。
|
|
||||||
|
|
||||||
## 2、使用指南
|
|
||||||
|
|
||||||
Task模块基于Logi-Job的广播模式与抢占模式，分别实现了任务的抢占执行、重复执行以及均衡执行，他们之间的差别是：
|
|
||||||
|
|
||||||
+ 抢占执行：同一个KS集群，同一个任务执行周期中，仅有一台KS主机执行该任务；
|
|
||||||
+ 重复执行：同一个KS集群，同一个任务执行周期中，所有KS主机都执行该任务。比如3台KS主机，3个Kafka集群，此时每台KS主机都会去采集这3个Kafka集群的指标；
|
|
||||||
+ 均衡执行：同一个KS集群，同一个任务执行周期中，每台KS主机仅执行该任务的一部分，所有的KS主机共同协作完成了任务。比如3台KS主机，3个Kafka集群，稳定运行情况下，每台KS主机将仅采集1个Kafka集群的指标，3台KS主机共同完成3个Kafka集群指标的采集。
|
|
||||||
|
|
||||||
下面我们看一下具体例子。
|
|
||||||
|
|
||||||
### 2.1、抢占模式——抢占执行
|
|
||||||
|
|
||||||
功能说明：
|
|
||||||
|
|
||||||
+ 同一个KS集群，同一个任务执行周期中，仅有一台KS主机执行该任务。
|
|
||||||
|
|
||||||
代码例子：
|
|
||||||
|
|
||||||
```java
|
|
||||||
// 1、实现Job接口，重写excute方法；
|
|
||||||
// 2、在类上添加@Task注解，并且配置好信息，指定为随机抢占模式；
|
|
||||||
// 效果：KS集群中，每5秒，会有一台KS主机输出 "测试定时任务运行中"；
|
|
||||||
@Task(name = "TestJob",
|
|
||||||
description = "测试定时任务",
|
|
||||||
cron = "*/5 * * * * ?",
|
|
||||||
autoRegister = true,
|
|
||||||
consensual = ConsensualEnum.RANDOM, // 这里一定要设置为RANDOM
|
|
||||||
timeout = 6 * 60)
|
|
||||||
public class TestJob implements Job {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public TaskResult execute(JobContext jobContext) throws Exception {
|
|
||||||
|
|
||||||
System.out.println("测试定时任务运行中");
|
|
||||||
return new TaskResult();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.2、广播模式——重复执行
|
|
||||||
|
|
||||||
功能说明：
|
|
||||||
|
|
||||||
+ 同一个KS集群，同一个任务执行周期中，所有KS主机都执行该任务。比如3台KS主机，3个Kafka集群，此时每台KS主机都会去重复采集这3个Kafka集群的指标。
|
|
||||||
|
|
||||||
代码例子：
|
|
||||||
|
|
||||||
```java
|
|
||||||
// 1、实现Job接口，重写excute方法；
|
|
||||||
// 2、在类上添加@Task注解，并且配置好信息，指定为广播抢占模式；
|
|
||||||
// 效果：KS集群中，每5秒，每台KS主机都会输出 "测试定时任务运行中"；
|
|
||||||
@Task(name = "TestJob",
|
|
||||||
description = "测试定时任务",
|
|
||||||
cron = "*/5 * * * * ?",
|
|
||||||
autoRegister = true,
|
|
||||||
consensual = ConsensualEnum.BROADCAST, // 这里一定要设置为BROADCAST
|
|
||||||
timeout = 6 * 60)
|
|
||||||
public class TestJob implements Job {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public TaskResult execute(JobContext jobContext) throws Exception {
|
|
||||||
|
|
||||||
System.out.println("测试定时任务运行中");
|
|
||||||
return new TaskResult();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.3、广播模式——均衡执行
|
|
||||||
|
|
||||||
功能说明：
|
|
||||||
|
|
||||||
+ 同一个KS集群，同一个任务执行周期中，每台KS主机仅执行该任务的一部分，所有的KS主机共同协作完成了任务。比如3台KS主机，3个Kafka集群，稳定运行情况下，每台KS主机将仅采集1个Kafka集群的指标，3台KS主机共同完成3个Kafka集群指标的采集。
|
|
||||||
|
|
||||||
代码例子：
|
|
||||||
|
|
||||||
+ 该模式有点特殊，是KS基于Logi-Job的广播模式，做的一个扩展，以下为一个使用例子：
|
|
||||||
|
|
||||||
```java
|
|
||||||
// 1、继承AbstractClusterPhyDispatchTask，实现processSubTask方法；
|
|
||||||
// 2、在类上添加@Task注解，并且配置好信息，指定为广播模式；
|
|
||||||
// 效果：在本样例中，每隔1分钟ks会将所有的kafka集群列表在ks集群主机内均衡拆分，每台主机会将分发到自身的Kafka集群依次执行processSubTask方法,实现KS集群的任务协同处理。
|
|
||||||
@Task(name = "kmJobTask",
|
|
||||||
description = "km job 模块调度执行任务",
|
|
||||||
cron = "0 0/1 * * * ? *",
|
|
||||||
autoRegister = true,
|
|
||||||
consensual = ConsensualEnum.BROADCAST,
|
|
||||||
timeout = 6 * 60)
|
|
||||||
public class KMJobTask extends AbstractClusterPhyDispatchTask {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private JobService jobService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected TaskResult processSubTask(ClusterPhy clusterPhy, long triggerTimeUnitMs) throws Exception {
|
|
||||||
jobService.scheduleJobByClusterId(clusterPhy.getId());
|
|
||||||
return TaskResult.SUCCESS;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 3、原理简介
|
|
||||||
|
|
||||||
### 3.1、Task注解说明
|
|
||||||
|
|
||||||
```java
|
|
||||||
public @interface Task {
|
|
||||||
String name() default ""; //任务名称
|
|
||||||
String description() default ""; //任务描述
|
|
||||||
String owner() default "system"; //拥有者
|
|
||||||
String cron() default ""; //定时执行的时间策略
|
|
||||||
int retryTimes() default 0; //失败以后所能重试的最大次数
|
|
||||||
long timeout() default 0; //在超时时间里重试
|
|
||||||
//是否自动注册任务到数据库中
|
|
||||||
//如果设置为false，需要手动去数据库km_task表注册定时任务信息。数据库记录和@Task注解缺一不可
|
|
||||||
boolean autoRegister() default false;
|
|
||||||
//执行模式：广播、随机抢占
|
|
||||||
//广播模式：同一集群下的所有服务器都会执行该定时任务
|
|
||||||
//随机抢占模式：同一集群下随机一台服务器执行该任务
|
|
||||||
ConsensualEnum consensual() default ConsensualEnum.RANDOM;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3.2、数据库表介绍
|
|
||||||
|
|
||||||
+ logi_task：记录项目中的定时任务信息，一个定时任务对应一条记录。
|
|
||||||
+ logi_job：具体任务执行信息。
|
|
||||||
+ logi_job_log：定时任务的执行日志。
|
|
||||||
+ logi_worker：记录机器信息，实现集群控制。
|
|
||||||
|
|
||||||
### 3.3、均衡执行简介
|
|
||||||
|
|
||||||
#### 3.3.1、类关系图
|
|
||||||
|
|
||||||
这里以KMJobTask为例，简单介绍KM中的定时任务实现逻辑。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
+ Job：使用logi组件实现定时任务，必须实现该接口。
|
|
||||||
+ Comparable & EntufyIdInterface：比较接口，实现任务的排序逻辑。
|
|
||||||
+ AbstractDispatchTask：实现广播模式下，任务的均衡分发。
|
|
||||||
+ AbstractClusterPhyDispatchTask：对分发到当前服务器的集群列表进行枚举。
|
|
||||||
+ KMJobTask:实现对单个集群的定时任务处理。
|
|
||||||
|
|
||||||
#### 3.3.2、关键类代码
|
|
||||||
|
|
||||||
+ **AbstractDispatchTask类**
|
|
||||||
|
|
||||||
```java
|
|
||||||
// 实现Job接口的抽象类，进行任务的负载均衡执行
|
|
||||||
public abstract class AbstractDispatchTask<E extends Comparable & EntifyIdInterface> implements Job {
|
|
||||||
|
|
||||||
// 罗列所有的任务
|
|
||||||
protected abstract List<E> listAllTasks();
|
|
||||||
|
|
||||||
// 执行被分配给该KS主机的任务
|
|
||||||
protected abstract TaskResult processTask(List<E> subTaskList, long triggerTimeUnitMs);
|
|
||||||
|
|
||||||
// 被Logi-Job触发执行该方法
|
|
||||||
// 该方法进行任务的分配
|
|
||||||
@Override
|
|
||||||
public TaskResult execute(JobContext jobContext) {
|
|
||||||
try {
|
|
||||||
|
|
||||||
long triggerTimeUnitMs = System.currentTimeMillis();
|
|
||||||
|
|
||||||
// 获取所有的任务
|
|
||||||
List<E> allTaskList = this.listAllTasks();
|
|
||||||
|
|
||||||
// 计算当前KS机器需要执行的任务
|
|
||||||
List<E> subTaskList = this.selectTask(allTaskList, jobContext.getAllWorkerCodes(), jobContext.getCurrentWorkerCode());
|
|
||||||
|
|
||||||
// 进行任务处理
|
|
||||||
return this.processTask(subTaskList, triggerTimeUnitMs);
|
|
||||||
} catch (Exception e) {
|
|
||||||
// ...
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
+ **AbstractClusterPhyDispatchTask类**
|
|
||||||
|
|
||||||
```java
|
|
||||||
// 继承AbstractDispatchTask的抽象类，对Kafka集群进行负载均衡执行
|
|
||||||
public abstract class AbstractClusterPhyDispatchTask extends AbstractDispatchTask<ClusterPhy> {
|
|
||||||
|
|
||||||
// 执行被分配的任务，具体由子类实现
|
|
||||||
protected abstract TaskResult processSubTask(ClusterPhy clusterPhy, long triggerTimeUnitMs) throws Exception;
|
|
||||||
|
|
||||||
// 返回所有的Kafka集群
|
|
||||||
@Override
|
|
||||||
public List<ClusterPhy> listAllTasks() {
|
|
||||||
return clusterPhyService.listAllClusters();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 执行被分配给该KS主机的Kafka集群任务
|
|
||||||
@Override
|
|
||||||
public TaskResult processTask(List<ClusterPhy> subTaskList, long triggerTimeUnitMs) { // ... }
|
|
||||||
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
+ **KMJobTask类**
|
|
||||||
|
|
||||||
```java
|
|
||||||
// 加上@Task注解，并配置任务执行信息
|
|
||||||
@Task(name = "kmJobTask",
|
|
||||||
description = "km job 模块调度执行任务",
|
|
||||||
cron = "0 0/1 * * * ? *",
|
|
||||||
autoRegister = true,
|
|
||||||
consensual = ConsensualEnum.BROADCAST,
|
|
||||||
timeout = 6 * 60)
|
|
||||||
// 继承AbstractClusterPhyDispatchTask类
|
|
||||||
public class KMJobTask extends AbstractClusterPhyDispatchTask {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private JobService jobService;
|
|
||||||
|
|
||||||
// 执行该Kafka集群的Job模块的任务
|
|
||||||
@Override
|
|
||||||
protected TaskResult processSubTask(ClusterPhy clusterPhy, long triggerTimeUnitMs) throws Exception {
|
|
||||||
jobService.scheduleJobByClusterId(clusterPhy.getId());
|
|
||||||
return TaskResult.SUCCESS;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 3.3.3、均衡执行总结
|
|
||||||
|
|
||||||
均衡执行的实现原理总结起来就是以下几点：
|
|
||||||
|
|
||||||
+ Logi-Job设置为广播模式，触发所有的KS主机执行任务；
|
|
||||||
+ 每台KS主机，被触发执行后，按照统一的规则，对任务列表，KS集群主机列表进行排序。然后按照顺序将任务列表均衡的分配给排序后的KS集群主机。KS集群稳定运行情况下，这一步保证了每台KS主机之间分配到的任务列表不重复，不丢失。
|
|
||||||
+ 最后每台KS主机，执行被分配到的任务。
|
|
||||||
|
|
||||||
## 4、注意事项
|
|
||||||
|
|
||||||
+ 不能100%保证任务在一个周期内，且仅且执行一次，可能出现重复执行或丢失的情况，所以必须严格是且仅且执行一次的任务，不建议基于Logi-Job进行任务控制。
|
|
||||||
+ 尽量让Logi-Job仅负责任务的触发，后续的执行建议放到自己创建的线程池中进行。
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 63 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 306 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 306 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 17 KiB |
@@ -1,43 +0,0 @@
|
|||||||
|
|
||||||
## 4.2、Kafka 多版本兼容方案
|
|
||||||
|
|
||||||
  当前 KnowStreaming 支持纳管多个版本的 kafka 集群，由于不同版本的 kafka 在指标采集、接口查询、行为操作上有些不一致，因此 KnowStreaming 需要一套机制来解决多 kafka 版本的纳管兼容性问题。
|
|
||||||
|
|
||||||
### 4.2.1、整体思路
|
|
||||||
|
|
||||||
  由于需要纳管多个 kafka 版本，而且未来还可能会纳管非 kafka 官方的版本，kafka 的版本号会存在着多种情况，所以首先要明确一个核心思想：KnowStreaming 提供尽可能多的纳管能力，但是不提供无限的纳管能力，每一个版本的 KnowStreaming 只纳管其自身声明的 kafka 版本，后续随着 KnowStreaming 自身版本的迭代，会逐步支持更多 kafka 版本的纳管接入。
|
|
||||||
|
|
||||||
### 4.2.2、构建版本兼容列表
|
|
||||||
|
|
||||||
  每一个版本的 KnowStreaming 都声明一个自身支持纳管的 kafka 版本列表，并且对 kafka 的版本号进行归一化处理，后续所有 KnowStreaming 对不同 kafka 集群的操作都和这个集群对应的版本号严格相关。
|
|
||||||
|
|
||||||
  KnowStreaming 对外提供自身所支持的 kafka 版本兼容列表，用以声明自身支持的版本范围。
|
|
||||||
|
|
||||||
  对于在集群接入过程中，如果希望接入当前 KnowStreaming 不支持的 kafka 版本的集群，KnowStreaming 建议在于的过程中选择相近的版本号接入。
|
|
||||||
|
|
||||||
### 4.2.3、构建版本兼容性字典
|
|
||||||
|
|
||||||
  在构建了 KnowStreaming 支持的 kafka 版本列表的基础上，KnowStreaming 在实现过程中，还会声明自身支持的所有兼容性，构建兼容性字典。
|
|
||||||
|
|
||||||
  当前 KnowStreaming 支持的 kafka 版本兼容性字典包括三个维度：
|
|
||||||
|
|
||||||
- 指标采集：同一个指标在不同 kafka 版本下可能获取的方式不一样，不同版本的 kafka 可能会有不同的指标，因此对于指标采集的处理需要构建兼容性字典。
|
|
||||||
- kafka api：同一个 kafka 的操作处理的方式在不同 kafka 版本下可能存在不一致，如：topic 的创建，因此 KnowStreaming 针对不同 kafka-api 的处理需要构建兼容性字典。
|
|
||||||
- 平台操作：KnowStreaming 在接入不同版本的 kafka 集群的时候，在平台页面上会根据不同的 kafka 版。
|
|
||||||
|
|
||||||
兼容性字典的核心设计字段如下：
|
|
||||||
|
|
||||||
| 兼容性维度 | 兼容项名称 | 最小 Kafka 版本号（归一化） | 最大 Kafka 版本号（归一化） | 处理器 |
|
|
||||||
| ---------- | ---------- | --------------------------- | --------------------------- | ------ |
|
|
||||||
|
|
||||||
KS-KM 根据其需要纳管的 kafka 版本，按照上述三个维度构建了完善了兼容性字典。
|
|
||||||
|
|
||||||
### 4.2.4、兼容性问题
|
|
||||||
|
|
||||||
  KS-KM 的每个版本针对需要纳管的 kafka 版本列表，事先分析各个版本的差异性和产品需求，同时 KS-KM 构建了一套专门处理兼容性的服务，来进行兼容性的注册、字典构建、处理器分发等操作，其中版本兼容性处理器是来具体处理不同 kafka 版本差异性的地方。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
  如上图所示，KS-KM 的 topic 服务在面对不同 kafka 版本时，其 topic 的创建、删除、扩容由于 kafka 版本自身的差异，导致 KnowStreaming 的处理也不一样，所以需要根据不同的 kafka 版本来实现不同的兼容性处理器，同时向 KnowStreaming 的兼容服务进行兼容性的注册，构建兼容性字典，后续在 KnowStreaming 的运行过程中，针对不同的 kafka 版本即可分发到不同的处理器中执行。
|
|
||||||
|
|
||||||
  后续随着 KnowStreaming 产品的发展，如果有新的兼容性的地方需要增加，只需要实现新版本的处理器，增加注册项即可。
|
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
## 3.3、指标说明
|
|
||||||
|
|
||||||
当前 KnowStreaming 支持针对 kafka 集群的多维度指标的采集和展示，同时也支持多个 kafka 版本的指标进行兼容，以下是 KnowStreaming 支持的指标说明。
|
|
||||||
|
|
||||||
现在对当前 KnowStreaming 支持的指标从指标名称、指标单位、指标说明、kafka 版本、企业/开源版指标 五个维度进行说明。
|
|
||||||
|
|
||||||
### 3.3.1、Cluster 指标
|
|
||||||
|
|
||||||
| 指标名称 | 指标单位 | 指标含义 | kafka 版本 | 企业/开源版指标 |
|
|
||||||
| ------------------------- | -------- |--------------------------------| ---------------- | --------------- |
|
|
||||||
| HealthScore | 分 | 集群总体的健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed | 个 | 集群总体健康检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal | 个 | 集群总体健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| HealthScore_Topics | 分 | 集群 Topics 的健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed_Topics | 个 | 集群 Topics 健康检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal_Topics | 个 | 集群 Topics 健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| HealthScore_Brokers | 分 | 集群 Brokers 的健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed_Brokers | 个 | 集群 Brokers 健康检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal_Brokers | 个 | 集群 Brokers 健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| HealthScore_Groups | 分 | 集群 Groups 的健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed_Groups | 个 | 集群 Groups 健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal_Groups | 个 | 集群 Groups 健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| HealthScore_Cluster | 分 | 集群自身的健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed_Cluster | 个 | 集群自身健康检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal_Cluster | 个 | 集群自身健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| TotalRequestQueueSize | 个 | 集群中总的请求队列数 | 全部版本 | 开源版 |
|
|
||||||
| TotalResponseQueueSize | 个 | 集群中总的响应队列数 | 全部版本 | 开源版 |
|
|
||||||
| EventQueueSize | 个 | 集群中 Controller 的 EventQueue 大小 | 2.0.0 及以上版本 | 开源版 |
|
|
||||||
| ActiveControllerCount | 个 | 集群中存活的 Controller 数 | 全部版本 | 开源版 |
|
|
||||||
| TotalProduceRequests | 个 | 集群中的 Produce 每秒请求数 | 全部版本 | 开源版 |
|
|
||||||
| TotalLogSize | byte | 集群总的已使用的磁盘大小 | 全部版本 | 开源版 |
|
|
||||||
| ConnectionsCount | 个 | 集群的连接(Connections)个数 | 全部版本 | 开源版 |
|
|
||||||
| Zookeepers | 个 | 集群中存活的 zk 节点个数 | 全部版本 | 开源版 |
|
|
||||||
| ZookeepersAvailable | 是/否 | ZK 地址是否合法 | 全部版本 | 开源版 |
|
|
||||||
| Brokers | 个 | 集群的 broker 的总数 | 全部版本 | 开源版 |
|
|
||||||
| BrokersAlive | 个 | 集群的 broker 的存活数 | 全部版本 | 开源版 |
|
|
||||||
| BrokersNotAlive | 个 | 集群的 broker 的未存活数 | 全部版本 | 开源版 |
|
|
||||||
| Replicas | 个 | 集群中 Replica 的总数 | 全部版本 | 开源版 |
|
|
||||||
| Topics | 个 | 集群中 Topic 的总数 | 全部版本 | 开源版 |
|
|
||||||
| Partitions | 个 | 集群的 Partitions 总数 | 全部版本 | 开源版 |
|
|
||||||
| PartitionNoLeader | 个 | 集群中的 PartitionNoLeader 总数 | 全部版本 | 开源版 |
|
|
||||||
| PartitionMinISR_S | 个 | 集群中的小于 PartitionMinISR 总数 | 全部版本 | 开源版 |
|
|
||||||
| PartitionMinISR_E | 个 | 集群中的等于 PartitionMinISR 总数 | 全部版本 | 开源版 |
|
|
||||||
| PartitionURP | 个 | 集群中的未同步的 Partition 总数 | 全部版本 | 开源版 |
|
|
||||||
| MessagesIn | 条/s | 集群每秒消息写入条数 | 全部版本 | 开源版 |
|
|
||||||
| Messages | 条 | 集群总的消息条数 | 全部版本 | 开源版 |
|
|
||||||
| LeaderMessages | 条 | 集群中 leader 总的消息条数 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn | byte/s | 集群的每秒写入字节数 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn_min_5 | byte/s | 集群的每秒写入字节数，5 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn_min_15 | byte/s | 集群的每秒写入字节数，15 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut | byte/s | 集群的每秒流出字节数 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut_min_5 | byte/s | 集群的每秒流出字节数，5 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut_min_15 | byte/s | 集群的每秒流出字节数，15 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| Groups | 个 | 集群中 Group 的总数 | 全部版本 | 开源版 |
|
|
||||||
| GroupActives | 个 | 集群中 ActiveGroup 的总数 | 全部版本 | 开源版 |
|
|
||||||
| GroupEmptys | 个 | 集群中 EmptyGroup 的总数 | 全部版本 | 开源版 |
|
|
||||||
| GroupRebalances | 个 | 集群中 RebalanceGroup 的总数 | 全部版本 | 开源版 |
|
|
||||||
| GroupDeads | 个 | 集群中 DeadGroup 的总数 | 全部版本 | 开源版 |
|
|
||||||
| Alive | 是/否 | 集群是否存活，1：存活；0：没有存活 | 全部版本 | 开源版 |
|
|
||||||
| AclEnable | 是/否 | 集群是否开启 Acl，1：是；0：否 | 全部版本 | 开源版 |
|
|
||||||
| Acls | 个 | ACL 数 | 全部版本 | 开源版 |
|
|
||||||
| AclUsers | 个 | ACL-KafkaUser 数 | 全部版本 | 开源版 |
|
|
||||||
| AclTopics | 个 | ACL-Topic 数 | 全部版本 | 开源版 |
|
|
||||||
| AclGroups | 个 | ACL-Group 数 | 全部版本 | 开源版 |
|
|
||||||
| Jobs | 个 | 集群任务总数 | 全部版本 | 开源版 |
|
|
||||||
| JobsRunning | 个 | 集群 running 任务总数 | 全部版本 | 开源版 |
|
|
||||||
| JobsWaiting | 个 | 集群 waiting 任务总数 | 全部版本 | 开源版 |
|
|
||||||
| JobsSuccess | 个 | 集群 success 任务总数 | 全部版本 | 开源版 |
|
|
||||||
| JobsFailed | 个 | 集群 failed 任务总数 | 全部版本 | 开源版 |
|
|
||||||
| LoadReBalanceEnable | 是/否 | 是否开启均衡, 1：是；0：否 | 全部版本 | 企业版 |
|
|
||||||
| LoadReBalanceCpu | 是/否 | CPU 是否均衡, 1：是；0：否 | 全部版本 | 企业版 |
|
|
||||||
| LoadReBalanceNwIn | 是/否 | BytesIn 是否均衡, 1：是；0：否 | 全部版本 | 企业版 |
|
|
||||||
| LoadReBalanceNwOut | 是/否 | BytesOut 是否均衡, 1：是；0：否 | 全部版本 | 企业版 |
|
|
||||||
| LoadReBalanceDisk | 是/否 | Disk 是否均衡, 1：是；0：否 | 全部版本 | 企业版 |
|
|
||||||
|
|
||||||
### 3.3.2、Broker 指标
|
|
||||||
|
|
||||||
| 指标名称 | 指标单位 | 指标含义 | kafka 版本 | 企业/开源版指标 |
|
|
||||||
| ----------------------- | -------- | ------------------------------------- | ---------- | --------------- |
|
|
||||||
| HealthScore | 分 | Broker 健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed | 个 | Broker 健康检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal | 个 | Broker 健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| TotalRequestQueueSize | 个 | Broker 的请求队列大小 | 全部版本 | 开源版 |
|
|
||||||
| TotalResponseQueueSize | 个 | Broker 的应答队列大小 | 全部版本 | 开源版 |
|
|
||||||
| ReplicationBytesIn | byte/s | Broker 的副本流入流量 | 全部版本 | 开源版 |
|
|
||||||
| ReplicationBytesOut | byte/s | Broker 的副本流出流量 | 全部版本 | 开源版 |
|
|
||||||
| MessagesIn | 条/s | Broker 的每秒消息流入条数 | 全部版本 | 开源版 |
|
|
||||||
| TotalProduceRequests | 个/s | Broker 上 Produce 的每秒请求数 | 全部版本 | 开源版 |
|
|
||||||
| NetworkProcessorAvgIdle | % | Broker 的网络处理器的空闲百分比 | 全部版本 | 开源版 |
|
|
||||||
| RequestHandlerAvgIdle | % | Broker 上请求处理器的空闲百分比 | 全部版本 | 开源版 |
|
|
||||||
| PartitionURP | 个 | Broker 上的未同步的副本的个数 | 全部版本 | 开源版 |
|
|
||||||
| ConnectionsCount | 个 | Broker 上网络链接的个数 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn | byte/s | Broker 的每秒数据写入量 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn_min_5 | byte/s | Broker 的每秒数据写入量，5 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn_min_15 | byte/s | Broker 的每秒数据写入量，15 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut | byte/s | Broker 的每秒数据流出量 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut_min_5 | byte/s | Broker 的每秒数据流出量，5 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut_min_15 | byte/s | Broker 的每秒数据流出量，15 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| ReassignmentBytesIn | byte/s | Broker 的每秒数据迁移写入量 | 全部版本 | 开源版 |
|
|
||||||
| ReassignmentBytesOut | byte/s | Broker 的每秒数据迁移流出量 | 全部版本 | 开源版 |
|
|
||||||
| Partitions | 个 | Broker 上的 Partition 个数 | 全部版本 | 开源版 |
|
|
||||||
| PartitionsSkew | % | Broker 上的 Partitions 倾斜度 | 全部版本 | 开源版 |
|
|
||||||
| Leaders | 个 | Broker 上的 Leaders 个数 | 全部版本 | 开源版 |
|
|
||||||
| LeadersSkew | % | Broker 上的 Leaders 倾斜度 | 全部版本 | 开源版 |
|
|
||||||
| LogSize | byte | Broker 上的消息容量大小 | 全部版本 | 开源版 |
|
|
||||||
| Alive | 是/否 | Broker 是否存活，1：存活；0：没有存活 | 全部版本 | 开源版 |
|
|
||||||
|
|
||||||
### 3.3.3、Topic 指标
|
|
||||||
|
|
||||||
| 指标名称 | 指标单位 | 指标含义 | kafka 版本 | 企业/开源版指标 |
|
|
||||||
| --------------------- | -------- | ------------------------------------- | ---------- | --------------- |
|
|
||||||
| HealthScore | 分 | 健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed | 个 | 健康项检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal | 个 | 健康项检查总数 | 全部版本 | 开源版 |
|
|
||||||
| TotalProduceRequests | 条/s | Topic 的 TotalProduceRequests | 全部版本 | 开源版 |
|
|
||||||
| BytesRejected | 个/s | Topic 的每秒写入拒绝量 | 全部版本 | 开源版 |
|
|
||||||
| FailedFetchRequests | 个/s | Topic 的 FailedFetchRequests | 全部版本 | 开源版 |
|
|
||||||
| FailedProduceRequests | 个/s | Topic 的 FailedProduceRequests | 全部版本 | 开源版 |
|
|
||||||
| ReplicationCount | 个 | Topic 总的副本数 | 全部版本 | 开源版 |
|
|
||||||
| Messages | 条 | Topic 总的消息数 | 全部版本 | 开源版 |
|
|
||||||
| MessagesIn | 条/s | Topic 每秒消息条数 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn | byte/s | Topic 每秒消息写入字节数 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn_min_5 | byte/s | Topic 每秒消息写入字节数，5 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn_min_15 | byte/s | Topic 每秒消息写入字节数，15 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut | byte/s | Topic 每秒消息流出字节数 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut_min_5 | byte/s | Topic 每秒消息流出字节数，5 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut_min_15 | byte/s | Topic 每秒消息流出字节数，15 分钟均值 | 全部版本 | 开源版 |
|
|
||||||
| LogSize | byte | Topic 的大小 | 全部版本 | 开源版 |
|
|
||||||
| PartitionURP | 个 | Topic 未同步的副本数 | 全部版本 | 开源版 |
|
|
||||||
|
|
||||||
### 3.3.4、Partition 指标
|
|
||||||
|
|
||||||
| 指标名称 | 指标单位 | 指标含义 | kafka 版本 | 企业/开源版指标 |
|
|
||||||
| -------------- | -------- | ----------------------------------------- | ---------- | --------------- |
|
|
||||||
| LogEndOffset | 条 | Partition 中 leader 副本的 LogEndOffset | 全部版本 | 开源版 |
|
|
||||||
| LogStartOffset | 条 | Partition 中 leader 副本的 LogStartOffset | 全部版本 | 开源版 |
|
|
||||||
| Messages | 条 | Partition 总的消息数 | 全部版本 | 开源版 |
|
|
||||||
| BytesIn | byte/s | Partition 的每秒消息流入字节数 | 全部版本 | 开源版 |
|
|
||||||
| BytesOut | byte/s | Partition 的每秒消息流出字节数 | 全部版本 | 开源版 |
|
|
||||||
| LogSize | byte | Partition 的大小 | 全部版本 | 开源版 |
|
|
||||||
|
|
||||||
### 3.3.5、Group 指标
|
|
||||||
|
|
||||||
| 指标名称 | 指标单位 | 指标含义 | kafka 版本 | 企业/开源版指标 |
|
|
||||||
| ----------------- | -------- | -------------------------- | ---------- | --------------- |
|
|
||||||
| HealthScore | 分 | 健康分 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckPassed | 个 | 健康检查通过数 | 全部版本 | 开源版 |
|
|
||||||
| HealthCheckTotal | 个 | 健康检查总数 | 全部版本 | 开源版 |
|
|
||||||
| OffsetConsumed | 条 | Consumer 的 CommitedOffset | 全部版本 | 开源版 |
|
|
||||||
| LogEndOffset | 条 | Consumer 的 LogEndOffset | 全部版本 | 开源版 |
|
|
||||||
| Lag | 条 | Group 消费者的 Lag 数 | 全部版本 | 开源版 |
|
|
||||||
| State | 个 | Group 组的状态 | 全部版本 | 开源版 |
|
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# 接入 ZK 带认证的 Kafka 集群
|
|
||||||
|
|
||||||
- [接入 ZK 带认证的 Kafka 集群](#接入-zk-带认证的-kafka-集群)
|
|
||||||
- [1、简要说明](#1简要说明)
|
|
||||||
- [2、支持 Digest-MD5 认证](#2支持-digest-md5-认证)
|
|
||||||
- [3、支持 Kerberos 认证](#3支持-kerberos-认证)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 1、简要说明
|
|
||||||
|
|
||||||
- 1、当前 KnowStreaming 暂无页面可以直接配置 ZK 的认证信息，但是 KnowStreaming 的后端预留了 MySQL 的字段用于存储 ZK 的认证信息，用户可通过将认证信息存储至该字段，从而达到支持接入 ZK 带认证的 Kafka 集群。
|
|
||||||
|
|
||||||
|
|
||||||
- 2、该字段位于 MySQL 库 ks_km_physical_cluster 表中的 zk_properties 字段，该字段的格式是：
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"openSecure": false, # 是否开启认证，开启时配置为true
|
|
||||||
"sessionTimeoutUnitMs": 15000, # session超时时间
|
|
||||||
"requestTimeoutUnitMs": 5000, # request超时时间
|
|
||||||
"otherProps": { # 其他配置，认证信息主要配置在该位置
|
|
||||||
"zookeeper.sasl.clientconfig": "kafkaClusterZK1" # 例子，
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- 3、实际生效的代码位置
|
|
||||||
```java
|
|
||||||
// 代码位置：https://github.com/didi/KnowStreaming/blob/master/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/KafkaAdminZKClient.java
|
|
||||||
|
|
||||||
kafkaZkClient = KafkaZkClient.apply(
|
|
||||||
clusterPhy.getZookeeper(),
|
|
||||||
zkConfig.getOpenSecure(), // 是否开启认证，开启时配置为true
|
|
||||||
zkConfig.getSessionTimeoutUnitMs(), // session超时时间
|
|
||||||
zkConfig.getRequestTimeoutUnitMs(), // request超时时间
|
|
||||||
5,
|
|
||||||
Time.SYSTEM,
|
|
||||||
"KS-ZK-ClusterPhyId-" + clusterPhyId,
|
|
||||||
"KS-ZK-SessionExpireListener-clusterPhyId-" + clusterPhyId,
|
|
||||||
Option.apply("KS-ZK-ClusterPhyId-" + clusterPhyId),
|
|
||||||
Option.apply(this.getZKConfig(clusterPhyId, zkConfig.getOtherProps())) // 其他配置，认证信息主要配置在该位置
|
|
||||||
);
|
|
||||||
```
|
|
||||||
|
|
||||||
- 4、SQL例子
|
|
||||||
```sql
|
|
||||||
update ks_km_physical_cluster set zk_properties='{ "openSecure": true, "otherProps": { "zookeeper.sasl.clientconfig": "kafkaClusterZK1" } }' where id=集群1的ID;
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
- 5、zk_properties 字段不能覆盖所有的场景，所以实际使用过程中还可能需要在此基础之上，进行其他的调整。比如，`Digest-MD5 认证` 和 `Kerberos 认证` 都还需要修改启动脚本等。后续看能否通过修改 ZK 客户端的源码，使得 ZK 认证的相关配置能和 Kafka 认证的配置一样方便。
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## 2、支持 Digest-MD5 认证
|
|
||||||
|
|
||||||
1. 假设你有两个 Kafka 集群， 对应两个 ZK 集群；
|
|
||||||
2. 两个 ZK 集群的认证信息如下所示
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# ZK1集群的认证信息，这里的 kafkaClusterZK1 可以是随意的名称，只需要和后续数据库的配置对应上即可。
|
|
||||||
kafkaClusterZK1 {
|
|
||||||
org.apache.zookeeper.server.auth.DigestLoginModule required
|
|
||||||
username="zk1"
|
|
||||||
password="zk1-passwd";
|
|
||||||
};
|
|
||||||
|
|
||||||
# ZK2集群的认证信息，这里的 kafkaClusterZK2 可以是随意的名称，只需要和后续数据库的配置对应上即可。
|
|
||||||
kafkaClusterZK2 {
|
|
||||||
org.apache.zookeeper.server.auth.DigestLoginModule required
|
|
||||||
username="zk2"
|
|
||||||
password="zk2-passwd";
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
3. 将这两个ZK集群的认证信息存储到 `/xxx/zk_client_jaas.conf` 文件中，文件中的内容如下所示：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kafkaClusterZK1 {
|
|
||||||
org.apache.zookeeper.server.auth.DigestLoginModule required
|
|
||||||
username="zk1"
|
|
||||||
password="zk1-passwd";
|
|
||||||
};
|
|
||||||
|
|
||||||
kafkaClusterZK2 {
|
|
||||||
org.apache.zookeeper.server.auth.DigestLoginModule required
|
|
||||||
username="zk2"
|
|
||||||
password="zk2-passwd";
|
|
||||||
};
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
4. 修改 KnowStreaming 的启动脚本
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# `KnowStreaming/bin/startup.sh` 中的 47 行的 JAVA_OPT 中追加如下设置
|
|
||||||
|
|
||||||
-Djava.security.auth.login.config=/xxx/zk_client_jaas.conf
|
|
||||||
```
|
|
||||||
|
|
||||||
5. 修改 KnowStreaming 的表数据
|
|
||||||
|
|
||||||
```sql
|
|
||||||
# 这里的 kafkaClusterZK1 要和 /xxx/zk_client_jaas.conf 中的对应上
|
|
||||||
update ks_km_physical_cluster set zk_properties='{ "openSecure": true, "otherProps": { "zookeeper.sasl.clientconfig": "kafkaClusterZK1" } }' where id=集群1的ID;
|
|
||||||
|
|
||||||
update ks_km_physical_cluster set zk_properties='{ "openSecure": true, "otherProps": { "zookeeper.sasl.clientconfig": "kafkaClusterZK2" } }' where id=集群2的ID;
|
|
||||||
```
|
|
||||||
|
|
||||||
6. 重启 KnowStreaming
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
## 3、支持 Kerberos 认证
|
|
||||||
|
|
||||||
**第一步：查看用户在ZK的ACL**
|
|
||||||
|
|
||||||
假设我们使用的用户是 `kafka` 这个用户。
|
|
||||||
|
|
||||||
- 1、查看 server.properties 的配置的 zookeeper.connect 的地址；
|
|
||||||
- 2、使用 `zkCli.sh -serve zookeeper.connect的地址` 登录到ZK页面；
|
|
||||||
- 3、ZK页面上，执行命令 `getAcl /kafka` 查看 `kafka` 用户的权限；
|
|
||||||
|
|
||||||
此时，我们可以看到如下信息：
|
|
||||||
|
|
||||||
|
|
||||||
`kafka` 用户需要的权限是 `cdrwa`。如果用户没有 `cdrwa` 权限的话，需要创建用户并授权，授权命令为：`setAcl`
|
|
||||||
|
|
||||||
|
|
||||||
**第二步：创建Kerberos的keytab并修改 KnowStreaming 主机**
|
|
||||||
|
|
||||||
- 1、在 Kerberos 的域中创建 `kafka/_HOST` 的 `keytab`，并导出。例如：`kafka/dbs-kafka-test-8-53`；
|
|
||||||
- 2、导出 keytab 后上传到安装 KS 的机器的 `/etc/keytab` 下；
|
|
||||||
- 3、在 KS 机器上，执行 `kinit -kt zookeepe.keytab kafka/dbs-kafka-test-8-53` 看是否能进行 `Kerberos` 登录；
|
|
||||||
- 4、可以登录后，配置 `/opt/zookeeper.jaas` 文件，例子如下：
|
|
||||||
```bash
|
|
||||||
Client {
|
|
||||||
com.sun.security.auth.module.Krb5LoginModule required
|
|
||||||
useKeyTab=true
|
|
||||||
storeKey=false
|
|
||||||
serviceName="zookeeper"
|
|
||||||
keyTab="/etc/keytab/zookeeper.keytab"
|
|
||||||
principal="kafka/dbs-kafka-test-8-53@XXX.XXX.XXX";
|
|
||||||
};
|
|
||||||
```
|
|
||||||
- 5、需要配置 `KDC-Server` 对 `KnowStreaming` 的机器开通防火墙，并在KS的机器 `/etc/host/` 配置 `kdc-server` 的 `hostname`。并将 `krb5.conf` 导入到 `/etc` 下；
|
|
||||||
|
|
||||||
|
|
||||||
**第三步：修改 KnowStreaming 的配置**
|
|
||||||
|
|
||||||
- 1、修改数据库，开启ZK的认证
|
|
||||||
```sql
|
|
||||||
update ks_km_physical_cluster set zk_properties='{ "openSecure": true }' where id=集群1的ID;
|
|
||||||
```
|
|
||||||
|
|
||||||
- 2、在 `KnowStreaming/bin/startup.sh` 中的47行的JAVA_OPT中追加如下设置
|
|
||||||
```bash
|
|
||||||
-Dsun.security.krb5.debug=true -Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/zookeeper.jaas
|
|
||||||
```
|
|
||||||
|
|
||||||
- 3、重启KS集群后再 start.out 中看到如下信息，则证明Kerberos配置成功；
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
**第四步：补充说明**
|
|
||||||
|
|
||||||
- 1、多Kafka集群如果用的是一样的Kerberos域的话，只需在每个`ZK`中给`kafka`用户配置`crdwa`权限即可，这样集群初始化的时候`zkclient`是都可以认证；
|
|
||||||
- 2、多个Kerberos域暂时未适配；
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
## 6.1、本地源码启动手册
|
|
||||||
|
|
||||||
### 6.1.1、打包方式
|
|
||||||
|
|
||||||
`Know Streaming` 采用前后端分离的开发模式，使用 Maven 对项目进行统一的构建管理。maven 在打包构建过程中，会将前后端代码一并打包生成最终的安装包。
|
|
||||||
|
|
||||||
`Know Streaming` 除了使用安装包启动之外，还可以通过本地源码启动完整的带前端页面的项目，下面我们正式开始介绍本地源码如何启动 `Know Streaming`。
|
|
||||||
|
|
||||||
### 6.1.2、环境要求
|
|
||||||
|
|
||||||
**系统支持**
|
|
||||||
|
|
||||||
`windows7+`、`Linux`、`Mac`
|
|
||||||
|
|
||||||
**环境依赖**
|
|
||||||
|
|
||||||
- Maven 3.6.3
|
|
||||||
- Node v12.20.0
|
|
||||||
- Java 8+
|
|
||||||
- MySQL 5.7
|
|
||||||
- Idea
|
|
||||||
- Elasticsearch 7.6
|
|
||||||
- Git
|
|
||||||
|
|
||||||
### 6.1.3、环境初始化
|
|
||||||
|
|
||||||
安装好环境信息之后，还需要初始化 MySQL 与 Elasticsearch 信息，包括：
|
|
||||||
|
|
||||||
- 初始化 MySQL 表及数据
|
|
||||||
- 初始化 Elasticsearch 索引
|
|
||||||
|
|
||||||
具体见：[单机部署手册](../install_guide/单机部署手册.md) 中的最后一步，部署 KnowStreaming 服务中的初始化相关工作。
|
|
||||||
|
|
||||||
### 6.1.4、本地启动
|
|
||||||
|
|
||||||
**第一步：本地打包**
|
|
||||||
|
|
||||||
执行 `mvn install` 可对项目进行前后端同时进行打包，通过该命令，除了可以对后端进行打包之外，还可以将前端相关的静态资源文件也一并打包出来。
|
|
||||||
|
|
||||||
**第二步：修改配置**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# 修改 km-rest/src/main/resources/application.yml 中相关的配置
|
|
||||||
|
|
||||||
# 修改MySQL的配置，中间省略了一些非必需修改的配置
|
|
||||||
spring:
|
|
||||||
datasource:
|
|
||||||
know-streaming:
|
|
||||||
jdbc-url: 修改为实际MYSQL地址
|
|
||||||
username: 修改为实际MYSQL用户名
|
|
||||||
password: 修改为实际MYSQL密码
|
|
||||||
logi-job:
|
|
||||||
jdbc-url: 修改为实际MYSQL地址
|
|
||||||
username: 修改为实际MYSQL用户名
|
|
||||||
password: 修改为实际MYSQL密码
|
|
||||||
logi-security:
|
|
||||||
jdbc-url: 修改为实际MYSQL地址
|
|
||||||
username: 修改为实际MYSQL用户名
|
|
||||||
password: 修改为实际MYSQL密码
|
|
||||||
|
|
||||||
# 修改ES的配置，中间省略了一些非必需修改的配置
|
|
||||||
es.client.address: 修改为实际ES地址
|
|
||||||
```
|
|
||||||
|
|
||||||
**第三步：配置 IDEA**
|
|
||||||
|
|
||||||
`Know Streaming`的 Main 方法在：
|
|
||||||
|
|
||||||
```java
|
|
||||||
km-rest/src/main/java/com/xiaojukeji/know/streaming/km/rest/KnowStreaming.java
|
|
||||||
```
|
|
||||||
|
|
||||||
IDEA 更多具体的配置如下图所示：
|
|
||||||
|
|
||||||
<p align="center">
|
|
||||||
<img src="http://img-ys011.didistatic.com/static/dc2img/do1_BW1RzgEMh4n6L4dL4ncl" width = "512" height = "318" div align=center />
|
|
||||||
</p>
|
|
||||||
|
|
||||||
**第四步：启动项目**
|
|
||||||
|
|
||||||
最后就是启动项目，在本地 console 中输出了 `KnowStreaming-KM started` 则表示我们已经成功启动 `Know Streaming` 了。
|
|
||||||
|
|
||||||
### 6.1.5、本地访问
|
|
||||||
|
|
||||||
`Know Streaming` 启动之后，可以访问一些信息，包括：
|
|
||||||
|
|
||||||
- 产品页面：http://localhost:8080 ，默认账号密码：`admin` / `admin2022_` 进行登录。`v3.0.0-beta.2`版本开始，默认账号密码为`admin` / `admin`；
|
|
||||||
- 接口地址：http://localhost:8080/swagger-ui.html 查看后端提供的相关接口。
|
|
||||||
|
|
||||||
更多信息，详见：[KnowStreaming 官网](https://knowstreaming.com/)
|
|
||||||
@@ -1,199 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 登录系统对接
|
|
||||||
|
|
||||||
[KnowStreaming](https://github.com/didi/KnowStreaming)（以下简称KS） 除了实现基于本地MySQL的用户登录认证方式外，还已经实现了基于Ldap的登录认证。
|
|
||||||
|
|
||||||
但是，登录认证系统并非仅此两种。因此，为了具有更好的拓展性，KS具有自定义登陆认证逻辑，快速对接已有系统的特性。
|
|
||||||
|
|
||||||
在KS中，我们将登陆认证相关的一些文件放在[km-extends](https://github.com/didi/KnowStreaming/tree/master/km-extends)模块下的[km-account](https://github.com/didi/KnowStreaming/tree/master/km-extends/km-account)模块里。
|
|
||||||
|
|
||||||
本文将介绍KS如何快速对接自有的用户登录认证系统。
|
|
||||||
|
|
||||||
### 对接步骤
|
|
||||||
|
|
||||||
- 创建一个登陆认证类，实现[LogiCommon](https://github.com/didi/LogiCommon)的LoginExtend接口；
|
|
||||||
- 将[application.yml](https://github.com/didi/KnowStreaming/blob/master/km-rest/src/main/resources/application.yml)中的spring.logi-security.login-extend-bean-name字段改为登陆认证类的bean名称；
|
|
||||||
|
|
||||||
```Java
|
|
||||||
//LoginExtend 接口
|
|
||||||
public interface LoginExtend {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证登录信息，同时记住登录状态
|
|
||||||
*/
|
|
||||||
UserBriefVO verifyLogin(AccountLoginDTO var1, HttpServletRequest var2, HttpServletResponse var3) throws LogiSecurityException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 登出接口，清楚登录状态
|
|
||||||
*/
|
|
||||||
Result<Boolean> logout(HttpServletRequest var1, HttpServletResponse var2);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 检查是否已经登录
|
|
||||||
*/
|
|
||||||
boolean interceptorCheck(HttpServletRequest var1, HttpServletResponse var2, String var3, List<String> var4) throws IOException;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 对接例子
|
|
||||||
|
|
||||||
我们以Ldap对接为例，说明KS如何对接登录认证系统。
|
|
||||||
|
|
||||||
+ 编写[LdapLoginServiceImpl](https://github.com/didi/KnowStreaming/blob/master/km-extends/km-account/src/main/java/com/xiaojukeji/know/streaming/km/account/login/ldap/LdapLoginServiceImpl.java)类，实现LoginExtend接口。
|
|
||||||
+ 设置[application.yml](https://github.com/didi/KnowStreaming/blob/master/km-rest/src/main/resources/application.yml)中的spring.logi-security.login-extend-bean-name=ksLdapLoginService。
|
|
||||||
|
|
||||||
完成上述两步即可实现KS对接Ldap认证登陆。
|
|
||||||
|
|
||||||
```Java
|
|
||||||
@Service("ksLdapLoginService")
|
|
||||||
public class LdapLoginServiceImpl implements LoginExtend {
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public UserBriefVO verifyLogin(AccountLoginDTO loginDTO,
|
|
||||||
HttpServletRequest request,
|
|
||||||
HttpServletResponse response) throws LogiSecurityException {
|
|
||||||
String decodePasswd = AESUtils.decrypt(loginDTO.getPw());
|
|
||||||
|
|
||||||
// 去LDAP验证账密
|
|
||||||
LdapPrincipal ldapAttrsInfo = ldapAuthentication.authenticate(loginDTO.getUserName(), decodePasswd);
|
|
||||||
if (ldapAttrsInfo == null) {
|
|
||||||
// 用户不存在，正常来说上如果有问题，上一步会直接抛出异常
|
|
||||||
throw new LogiSecurityException(ResultCode.USER_NOT_EXISTS);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 进行业务相关操作
|
|
||||||
|
|
||||||
// 记录登录状态，Ldap因为无法记录登录状态，因此有KnowStreaming进行记录
|
|
||||||
initLoginContext(request, response, loginDTO.getUserName(), user.getId());
|
|
||||||
return CopyBeanUtil.copy(user, UserBriefVO.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Boolean> logout(HttpServletRequest request, HttpServletResponse response) {
|
|
||||||
|
|
||||||
//清理cookie和session
|
|
||||||
|
|
||||||
return Result.buildSucc(Boolean.TRUE);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean interceptorCheck(HttpServletRequest request, HttpServletResponse response, String requestMappingValue, List<String> whiteMappingValues) throws IOException {
|
|
||||||
|
|
||||||
// 检查是否已经登录
|
|
||||||
String userName = HttpRequestUtil.getOperator(request);
|
|
||||||
if (StringUtils.isEmpty(userName)) {
|
|
||||||
// 未登录，则进行登出
|
|
||||||
logout(request, response);
|
|
||||||
return Boolean.FALSE;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Boolean.TRUE;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 实现原理
|
|
||||||
|
|
||||||
因为登陆和登出整体实现逻辑是一致的，所以我们以登陆逻辑为例进行介绍。
|
|
||||||
|
|
||||||
+ 登陆原理
|
|
||||||
|
|
||||||
登陆走的是[LogiCommon](https://github.com/didi/LogiCommon)自带的LoginController。
|
|
||||||
|
|
||||||
```java
|
|
||||||
@RestController
|
|
||||||
public class LoginController {
|
|
||||||
|
|
||||||
|
|
||||||
//登陆接口
|
|
||||||
@PostMapping({"/login"})
|
|
||||||
public Result<UserBriefVO> login(HttpServletRequest request, HttpServletResponse response, @RequestBody AccountLoginDTO loginDTO) {
|
|
||||||
try {
|
|
||||||
//登陆认证
|
|
||||||
UserBriefVO userBriefVO = this.loginService.verifyLogin(loginDTO, request, response);
|
|
||||||
return Result.success(userBriefVO);
|
|
||||||
|
|
||||||
} catch (LogiSecurityException var5) {
|
|
||||||
return Result.fail(var5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
而登陆操作是调用LoginServiceImpl类来实现，但是具体由哪个登陆认证类来执行登陆操作却由loginExtendBeanTool来指定。
|
|
||||||
|
|
||||||
```java
|
|
||||||
//LoginServiceImpl类
|
|
||||||
@Service
|
|
||||||
public class LoginServiceImpl implements LoginService {
|
|
||||||
|
|
||||||
//实现登陆操作，但是具体哪个登陆类由loginExtendBeanTool来管理
|
|
||||||
public UserBriefVO verifyLogin(AccountLoginDTO loginDTO, HttpServletRequest request, HttpServletResponse response) throws LogiSecurityException {
|
|
||||||
|
|
||||||
return this.loginExtendBeanTool.getLoginExtendImpl().verifyLogin(loginDTO, request, response);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
而loginExtendBeanTool类会优先去查找用户指定的登陆认证类，如果失败则调用默认的登陆认证函数。
|
|
||||||
|
|
||||||
```java
|
|
||||||
//LoginExtendBeanTool类
|
|
||||||
@Component("logiSecurityLoginExtendBeanTool")
|
|
||||||
public class LoginExtendBeanTool {
|
|
||||||
|
|
||||||
public LoginExtend getLoginExtendImpl() {
|
|
||||||
LoginExtend loginExtend;
|
|
||||||
//先调用用户指定登陆类，如果失败则调用系统默认登陆认证
|
|
||||||
try {
|
|
||||||
//调用的类由spring.logi-security.login-extend-bean-name指定
|
|
||||||
loginExtend = this.getCustomLoginExtendImplBean();
|
|
||||||
} catch (UnsupportedOperationException var3) {
|
|
||||||
loginExtend = this.getDefaultLoginExtendImplBean();
|
|
||||||
}
|
|
||||||
|
|
||||||
return loginExtend;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
+ 认证原理
|
|
||||||
|
|
||||||
认证的实现则比较简单，向Spring中注册我们的拦截器PermissionInterceptor。
|
|
||||||
|
|
||||||
拦截器会调用LoginServiceImpl类的拦截方法，LoginServiceImpl后续处理逻辑就和前面登陆是一致的。
|
|
||||||
|
|
||||||
```java
|
|
||||||
public class PermissionInterceptor implements HandlerInterceptor {
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 拦截预处理
|
|
||||||
* @return boolean false:拦截, 不向下执行, true:放行
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
|
|
||||||
|
|
||||||
//免登录相关校验，如果验证通过，提前返回
|
|
||||||
|
|
||||||
//走拦截函数，进行普通用户验证
|
|
||||||
return loginService.interceptorCheck(request, response, classRequestMappingValue, whiteMappingValues);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
@@ -1,276 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 2、解决连接 JMX 失败
|
|
||||||
|
|
||||||
- [2、解决连接 JMX 失败](#2解决连接-jmx-失败)
|
|
||||||
- [2.1、正异常现象](#21正异常现象)
|
|
||||||
- [2.2、异因一：JMX未开启](#22异因一jmx未开启)
|
|
||||||
- [2.2.1、异常现象](#221异常现象)
|
|
||||||
- [2.2.2、解决方案](#222解决方案)
|
|
||||||
- [2.3、异原二：JMX配置错误](#23异原二jmx配置错误)
|
|
||||||
- [2.3.1、异常现象](#231异常现象)
|
|
||||||
- [2.3.2、解决方案](#232解决方案)
|
|
||||||
- [2.4、异因三：JMX开启SSL](#24异因三jmx开启ssl)
|
|
||||||
- [2.4.1、异常现象](#241异常现象)
|
|
||||||
- [2.4.2、解决方案](#242解决方案)
|
|
||||||
- [2.5、异因四：连接了错误IP](#25异因四连接了错误ip)
|
|
||||||
- [2.5.1、异常现象](#251异常现象)
|
|
||||||
- [2.5.2、解决方案](#252解决方案)
|
|
||||||
- [2.6、异因五：连接了错误端口](#26异因五连接了错误端口)
|
|
||||||
- [2.6.1、异常现象](#261异常现象)
|
|
||||||
- [2.6.2、解决方案](#262解决方案)
|
|
||||||
|
|
||||||
|
|
||||||
背景：Kafka 通过 JMX 服务进行运行指标的暴露，因此 `KnowStreaming` 会主动连接 Kafka 的 JMX 服务进行指标采集。如果我们发现页面缺少指标，那么可能原因之一是 Kafka 的 JMX 端口配置的有问题导致指标获取失败，进而页面没有数据。
|
|
||||||
|
|
||||||
|
|
||||||
### 2.1、正异常现象
|
|
||||||
|
|
||||||
**1、异常现象**
|
|
||||||
|
|
||||||
Broker 列表的 JMX PORT 列出现红色感叹号，则表示 JMX 连接存在异常。
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_MLlLCfAktne4X6MBtBUd width="90%">
|
|
||||||
|
|
||||||
|
|
||||||
**2、正常现象**
|
|
||||||
|
|
||||||
Broker 列表的 JMX PORT 列出现绿色，则表示 JMX 连接正常。
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_ymtDTCiDlzfrmSCez2lx width="90%">
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.2、异因一：JMX未开启
|
|
||||||
|
|
||||||
#### 2.2.1、异常现象
|
|
||||||
|
|
||||||
broker列表的JMX Port值为-1，对应Broker的JMX未开启。
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_E1PD8tPsMeR2zYLFBFAu width="90%">
|
|
||||||
|
|
||||||
#### 2.2.2、解决方案
|
|
||||||
|
|
||||||
开启JMX，开启流程如下：
|
|
||||||
|
|
||||||
1、修改kafka的bin目录下面的：`kafka-server-start.sh`文件
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 在这个下面增加JMX端口的配置
|
|
||||||
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
|
|
||||||
export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
|
|
||||||
export JMX_PORT=9999 # 增加这个配置, 这里的数值并不一定是要9999
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
2、修改kafka的bin目录下面对的：`kafka-run-class.sh`文件
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# JMX settings
|
|
||||||
if [ -z "$KAFKA_JMX_OPTS" ]; then
|
|
||||||
KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=当前机器的IP"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# JMX port to use
|
|
||||||
if [ $JMX_PORT ]; then
|
|
||||||
KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
3、重启Kafka-Broker。
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.3、异原二：JMX配置错误
|
|
||||||
|
|
||||||
#### 2.3.1、异常现象
|
|
||||||
|
|
||||||
错误日志：
|
|
||||||
|
|
||||||
```log
|
|
||||||
# 错误一： 错误提示的是真实的IP，这样的话基本就是JMX配置的有问题了。
|
|
||||||
2021-01-27 10:06:20.730 ERROR 50901 --- [ics-Thread-1-62] c.x.k.m.c.utils.jmx.JmxConnectorWrap : JMX connect exception, host:192.168.0.1 port:9999. java.rmi.ConnectException: Connection refused to host: 192.168.0.1; nested exception is:
|
|
||||||
|
|
||||||
# 错误二：错误提示的是127.0.0.1这个IP，这个是机器的hostname配置的可能有问题。
|
|
||||||
2021-01-27 10:06:20.730 ERROR 50901 --- [ics-Thread-1-62] c.x.k.m.c.utils.jmx.JmxConnectorWrap : JMX connect exception, host:127.0.0.1 port:9999. java.rmi.ConnectException: Connection refused to host: 127.0.0.1;; nested exception is:
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 2.3.2、解决方案
|
|
||||||
|
|
||||||
开启JMX，开启流程如下：
|
|
||||||
|
|
||||||
1、修改kafka的bin目录下面的：`kafka-server-start.sh`文件
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 在这个下面增加JMX端口的配置
|
|
||||||
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
|
|
||||||
export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
|
|
||||||
export JMX_PORT=9999 # 增加这个配置, 这里的数值并不一定是要9999
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
2、修改kafka的bin目录下面对的：`kafka-run-class.sh`文件
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# JMX settings
|
|
||||||
if [ -z "$KAFKA_JMX_OPTS" ]; then
|
|
||||||
KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=当前机器的IP"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# JMX port to use
|
|
||||||
if [ $JMX_PORT ]; then
|
|
||||||
KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
3、重启Kafka-Broker。
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.4、异因三：JMX开启SSL
|
|
||||||
|
|
||||||
#### 2.4.1、异常现象
|
|
||||||
|
|
||||||
```log
|
|
||||||
# 连接JMX的日志中，出现SSL认证失败的相关日志。TODO：欢迎补充具体日志案例。
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 2.4.2、解决方案
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_kNyCi8H9wtHSRkWurB6S width="50%">
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### 2.5、异因四：连接了错误IP
|
|
||||||
|
|
||||||
#### 2.5.1、异常现象
|
|
||||||
|
|
||||||
Broker 配置了内外网，而JMX在配置时，可能配置了内网IP或者外网IP，此时`KnowStreaming` 需要连接到特定网络的IP才可以进行访问。
|
|
||||||
|
|
||||||
比如：Broker在ZK的存储结构如下所示，我们期望连接到 `endpoints` 中标记为 `INTERNAL` 的地址，但是 `KnowStreaming` 却连接了 `EXTERNAL` 的地址。
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"listener_security_protocol_map": {
|
|
||||||
"EXTERNAL": "SASL_PLAINTEXT",
|
|
||||||
"INTERNAL": "SASL_PLAINTEXT"
|
|
||||||
},
|
|
||||||
"endpoints": [
|
|
||||||
"EXTERNAL://192.168.0.1:7092",
|
|
||||||
"INTERNAL://192.168.0.2:7093"
|
|
||||||
],
|
|
||||||
"jmx_port": 8099,
|
|
||||||
"host": "192.168.0.1",
|
|
||||||
"timestamp": "1627289710439",
|
|
||||||
"port": -1,
|
|
||||||
"version": 4
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 2.5.2、解决方案
|
|
||||||
|
|
||||||
可以手动往`ks_km_physical_cluster`表的`jmx_properties`字段增加一个`useWhichEndpoint`字段，从而控制 `KnowStreaming` 连接到特定的JMX IP及PORT。
|
|
||||||
|
|
||||||
`jmx_properties`格式：
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"maxConn": 100, // KM对单台Broker的最大JMX连接数
|
|
||||||
"username": "xxxxx", //用户名，可以不填写
|
|
||||||
"password": "xxxx", // 密码，可以不填写
|
|
||||||
"openSSL": true, //开启SSL, true表示开启ssl, false表示关闭
|
|
||||||
"useWhichEndpoint": "EXTERNAL" //指定要连接的网络名称，填写EXTERNAL就是连接endpoints里面的EXTERNAL地址
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
SQL例子：
|
|
||||||
|
|
||||||
```sql
|
|
||||||
UPDATE ks_km_physical_cluster SET jmx_properties='{ "maxConn": 10, "username": "xxxxx", "password": "xxxx", "openSSL": false , "useWhichEndpoint": "xxx"}' where id={xxx};
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.6、异因五：连接了错误端口
|
|
||||||
|
|
||||||
3.3.0 以上版本，或者是 master 分支最新代码，才具备该能力。
|
|
||||||
|
|
||||||
#### 2.6.1、异常现象
|
|
||||||
|
|
||||||
在 AWS 或者是容器上的 Kafka-Broker，使用同一个IP，但是外部服务想要去连接 JMX 端口时，需要进行映射。因此 KnowStreaming 如果直接连接 ZK 上获取到的 JMX 端口，会连接失败，因此需要具备连接端口可配置的能力。
|
|
||||||
|
|
||||||
TODO：补充具体的日志。
|
|
||||||
|
|
||||||
|
|
||||||
#### 2.6.2、解决方案
|
|
||||||
|
|
||||||
可以手动往`ks_km_physical_cluster`表的`jmx_properties`字段增加一个`specifiedJmxPortList`字段，从而控制 `KnowStreaming` 连接到特定的JMX PORT。
|
|
||||||
|
|
||||||
`jmx_properties`格式：
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jmxPort": 2445, // 最低优先级使用的jmx端口
|
|
||||||
"maxConn": 100, // KM对单台Broker的最大JMX连接数
|
|
||||||
"username": "xxxxx", //用户名，可以不填写
|
|
||||||
"password": "xxxx", // 密码，可以不填写
|
|
||||||
"openSSL": true, //开启SSL, true表示开启ssl, false表示关闭
|
|
||||||
"useWhichEndpoint": "EXTERNAL", //指定要连接的网络名称，填写EXTERNAL就是连接endpoints里面的EXTERNAL地址
|
|
||||||
"specifiedJmxPortList": [ // 配置最高优先使用的jmx端口
|
|
||||||
{
|
|
||||||
"serverId": "1", // kafka-broker的brokerId, 注意这个是字符串类型，字符串类型的原因是要兼容connect的jmx端口的连接
|
|
||||||
"jmxPort": 1234 // 该 broker 所连接的jmx端口
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"serverId": "2",
|
|
||||||
"jmxPort": 1234
|
|
||||||
},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
SQL例子：
|
|
||||||
|
|
||||||
```sql
|
|
||||||
UPDATE ks_km_physical_cluster SET jmx_properties='{ "maxConn": 10, "username": "xxxxx", "password": "xxxx", "openSSL": false , "specifiedJmxPortList": [{"serverId": "1", "jmxPort": 1234}] }' where id={xxx};
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
@@ -1,183 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
# 页面无数据排查手册
|
|
||||||
|
|
||||||
- [页面无数据排查手册](#页面无数据排查手册)
|
|
||||||
- [1、集群接入错误](#1集群接入错误)
|
|
||||||
- [1.1、异常现象](#11异常现象)
|
|
||||||
- [1.2、解决方案](#12解决方案)
|
|
||||||
- [1.3、正常情况](#13正常情况)
|
|
||||||
- [2、JMX连接失败](#2jmx连接失败)
|
|
||||||
- [3、ElasticSearch问题](#3elasticsearch问题)
|
|
||||||
- [3.1、异因一：缺少索引](#31异因一缺少索引)
|
|
||||||
- [3.1.1、异常现象](#311异常现象)
|
|
||||||
- [3.1.2、解决方案](#312解决方案)
|
|
||||||
- [3.2、异因二：索引模板错误](#32异因二索引模板错误)
|
|
||||||
- [3.2.1、异常现象](#321异常现象)
|
|
||||||
- [3.2.2、解决方案](#322解决方案)
|
|
||||||
- [3.3、异因三：集群Shard满](#33异因三集群shard满)
|
|
||||||
- [3.3.1、异常现象](#331异常现象)
|
|
||||||
- [3.3.2、解决方案](#332解决方案)
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1、集群接入错误
|
|
||||||
|
|
||||||
### 1.1、异常现象
|
|
||||||
|
|
||||||
如下图所示，集群非空时，大概率为地址配置错误导致。
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_BRiXBvqYFK2dxSF1aqgZ width="80%">
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 1.2、解决方案
|
|
||||||
|
|
||||||
接入集群时，依据提示的错误，进行相应的解决。例如：
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_Yn4LhV8aeSEKX1zrrkUi width="50%">
|
|
||||||
|
|
||||||
### 1.3、正常情况
|
|
||||||
|
|
||||||
接入集群时，页面信息都自动正常出现，没有提示错误。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2、JMX连接失败
|
|
||||||
|
|
||||||
背景：Kafka 通过 JMX 服务进行运行指标的暴露，因此 `KnowStreaming` 会主动连接 Kafka 的 JMX 服务进行指标采集。如果我们发现页面缺少指标，那么可能原因之一是 Kafka 的 JMX 端口配置的有问题导致指标获取失败，进而页面没有数据。
|
|
||||||
|
|
||||||
|
|
||||||
具体见同目录下的文档：[解决连接JMX失败](./%E8%A7%A3%E5%86%B3%E8%BF%9E%E6%8E%A5JMX%E5%A4%B1%E8%B4%A5.md)
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 3、ElasticSearch问题
|
|
||||||
|
|
||||||
**背景：**
|
|
||||||
`KnowStreaming` 将从 Kafka 中采集到的指标存储到 ES 中，如果 ES 存在问题，则也可能会导致页面出现无数据的情况。
|
|
||||||
|
|
||||||
**日志：**
|
|
||||||
`KnowStreaming` 读写 ES 相关日志，在 `logs/es/es.log` 中！
|
|
||||||
|
|
||||||
|
|
||||||
**注意：**
|
|
||||||
mac系统在执行curl指令时，可能报zsh错误。可参考以下操作。
|
|
||||||
|
|
||||||
```bash
|
|
||||||
1 进入.zshrc 文件 vim ~/.zshrc
|
|
||||||
2.在.zshrc中加入 setopt no_nomatch
|
|
||||||
3.更新配置 source ~/.zshrc
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 3.1、异因一：缺少索引
|
|
||||||
|
|
||||||
#### 3.1.1、异常现象
|
|
||||||
|
|
||||||
报错信息
|
|
||||||
|
|
||||||
```log
|
|
||||||
# 日志位置 logs/es/es.log
|
|
||||||
com.didiglobal.logi.elasticsearch.client.model.exception.ESIndexNotFoundException: method [GET], host[http://127.0.0.1:9200], URI [/ks_kafka_broker_metric_2022-10-21,ks_kafka_broker_metric_2022-10-22/_search], status line [HTTP/1.1 404 Not Found]
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
`curl http://{ES的IP地址}:{ES的端口号}/_cat/indices/ks_kafka*` 查看KS索引列表，发现没有索引。
|
|
||||||
|
|
||||||
#### 3.1.2、解决方案
|
|
||||||
|
|
||||||
执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来创建索引及模版。
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### 3.2、异因二：索引模板错误
|
|
||||||
|
|
||||||
#### 3.2.1、异常现象
|
|
||||||
|
|
||||||
多集群列表有数据，集群详情页图标无数据。查询KS索引模板列表，发现不存在。
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl {ES的IP地址}:{ES的端口号}/_cat/templates/ks_kafka*?v&h=name
|
|
||||||
```
|
|
||||||
|
|
||||||
正常KS模板如下图所示。
|
|
||||||
|
|
||||||
<img src=http://img-ys011.didistatic.com/static/dc2img/do1_l79bPYSci9wr6KFwZDA6 width="90%">
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 3.2.2、解决方案
|
|
||||||
|
|
||||||
删除KS索引模板和索引
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -XDELETE {ES的IP地址}:{ES的端口号}/ks_kafka*
|
|
||||||
curl -XDELETE {ES的IP地址}:{ES的端口号}/_template/ks_kafka*
|
|
||||||
```
|
|
||||||
|
|
||||||
执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来创建索引及模版。
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### 3.3、异因三：集群Shard满
|
|
||||||
|
|
||||||
#### 3.3.1、异常现象
|
|
||||||
|
|
||||||
报错信息
|
|
||||||
|
|
||||||
```log
|
|
||||||
# 日志位置 logs/es/es.log
|
|
||||||
|
|
||||||
{"error":{"root_cause":[{"type":"validation_exception","reason":"Validation Failed: 1: this action would add [4] total shards, but this cluster currently has [1000]/[1000] maximum shards open;"}],"type":"validation_exception","reason":"Validation Failed: 1: this action would add [4] total shards, but this cluster currently has [1000]/[1000] maximum shards open;"},"status":400}
|
|
||||||
```
|
|
||||||
|
|
||||||
尝试手动创建索引失败。
|
|
||||||
|
|
||||||
```bash
|
|
||||||
#创建ks_kafka_cluster_metric_test索引的指令
|
|
||||||
curl -s -XPUT http://{ES的IP地址}:{ES的端口号}/ks_kafka_cluster_metric_test
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
#### 3.3.2、解决方案
|
|
||||||
|
|
||||||
ES索引的默认分片数量为1000，达到数量以后，索引创建失败。
|
|
||||||
|
|
||||||
+ 扩大ES索引数量上限，执行指令
|
|
||||||
|
|
||||||
```
|
|
||||||
curl -XPUT -H"content-type:application/json" http://{ES的IP地址}:{ES的端口号}/_cluster/settings -d '
|
|
||||||
{
|
|
||||||
"persistent": {
|
|
||||||
"cluster": {
|
|
||||||
"max_shards_per_node":{索引上限，默认为1000, 测试时可以将其调整为10000}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来补全索引。
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,417 +0,0 @@
|
|||||||
## 2.1、单机部署
|
|
||||||
|
|
||||||
**风险提示**
|
|
||||||
|
|
||||||
⚠️ 脚本全自动安装，会将所部署机器上的 MySQL、JDK、ES 等进行删除重装，请注意原有服务丢失风险。
|
|
||||||
|
|
||||||
### 2.1.1、安装说明
|
|
||||||
|
|
||||||
- 以 `v3.0.0-beta.1` 版本为例进行部署；
|
|
||||||
- 以 CentOS-7 为例，系统基础配置要求 4C-8G；
|
|
||||||
- 部署完成后，可通过浏览器：`IP:PORT` 进行访问，默认端口是 `8080`，系统默认账号密码: `admin` / `admin2022_`。
|
|
||||||
- `v3.0.0-beta.2`版本开始，默认账号密码为`admin` / `admin`；
|
|
||||||
- 本文为单机部署，如需分布式部署，[请联系我们](https://knowstreaming.com/support-center)
|
|
||||||
|
|
||||||
**软件依赖**
|
|
||||||
|
|
||||||
| 软件名 | 版本要求 | 默认端口 |
|
|
||||||
| ------------- | ------------ | -------- |
|
|
||||||
| MySQL | v5.7 或 v8.0 | 3306 |
|
|
||||||
| ElasticSearch | v7.6+ | 8060 |
|
|
||||||
| JDK | v8+ | - |
|
|
||||||
| CentOS | v6+ | - |
|
|
||||||
| Ubuntu | v16+ | - |
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.1.2、脚本部署
|
|
||||||
|
|
||||||
**在线安装**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 在服务器中下载安装脚本, 该脚本中会在当前目录下，重新安装MySQL。重装后的mysql密码存放在当前目录的mysql.password文件中。
|
|
||||||
wget https://s3-gzpu.didistatic.com/pub/knowstreaming/deploy_KnowStreaming-3.0.0-beta.1.sh
|
|
||||||
|
|
||||||
# 执行脚本
|
|
||||||
sh deploy_KnowStreaming.sh
|
|
||||||
|
|
||||||
# 访问地址
|
|
||||||
127.0.0.1:8080
|
|
||||||
```
|
|
||||||
|
|
||||||
**离线安装**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 将安装包下载到本地且传输到目标服务器
|
|
||||||
wget https://s3-gzpu.didistatic.com/pub/knowstreaming/KnowStreaming-3.0.0-beta.1-offline.tar.gz
|
|
||||||
|
|
||||||
# 解压安装包
|
|
||||||
tar -zxf KnowStreaming-3.0.0-beta.1-offline.tar.gz
|
|
||||||
|
|
||||||
# 执行安装脚本
|
|
||||||
sh deploy_KnowStreaming-offline.sh
|
|
||||||
|
|
||||||
# 访问地址
|
|
||||||
127.0.0.1:8080
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.1.3、容器部署
|
|
||||||
|
|
||||||
#### 2.1.3.1、Helm
|
|
||||||
|
|
||||||
**环境依赖**
|
|
||||||
|
|
||||||
- Kubernetes >= 1.14 ，Helm >= 2.17.0
|
|
||||||
|
|
||||||
- 默认依赖全部安装，ElasticSearch（3 节点集群模式） + MySQL(单机) + KnowStreaming-manager + KnowStreaming-ui
|
|
||||||
|
|
||||||
- 使用已有的 ElasticSearch(7.6.x) 和 MySQL(5.7) 只需调整 values.yaml 部分参数即可
|
|
||||||
|
|
||||||
**安装命令**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 相关镜像在Docker Hub都可以下载
|
|
||||||
# 快速安装(NAMESPACE需要更改为已存在的，安装启动需要几分钟初始化请稍等~)
|
|
||||||
helm install -n [NAMESPACE] [NAME] http://download.knowstreaming.com/charts/knowstreaming-manager-0.1.5.tgz
|
|
||||||
|
|
||||||
# 获取KnowStreaming前端ui的service. 默认nodeport方式.
|
|
||||||
# (http://nodeIP:nodeport，默认用户名密码：admin/admin2022_)
|
|
||||||
# `v3.0.0-beta.2`版本开始（helm chart包版本0.1.4开始），默认账号密码为`admin` / `admin`；
|
|
||||||
|
|
||||||
# 添加仓库
|
|
||||||
helm repo add knowstreaming http://download.knowstreaming.com/charts
|
|
||||||
|
|
||||||
# 拉取最新版本
|
|
||||||
helm pull knowstreaming/knowstreaming-manager
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 2.1.3.2、Docker Compose
|
|
||||||
**环境依赖**
|
|
||||||
|
|
||||||
- [Docker](https://docs.docker.com/engine/install/)
|
|
||||||
- [Docker Compose](https://docs.docker.com/compose/install/)
|
|
||||||
|
|
||||||
|
|
||||||
**安装命令**
|
|
||||||
```bash
|
|
||||||
# `v3.0.0-beta.2`版本开始(docker镜像为0.2.0版本开始)，默认账号密码为`admin` / `admin`；
|
|
||||||
# https://hub.docker.com/u/knowstreaming 在此处寻找最新镜像版本
|
|
||||||
# mysql与es可以使用自己搭建的服务,调整对应配置即可
|
|
||||||
|
|
||||||
# 复制docker-compose.yml到指定位置后执行下方命令即可启动
|
|
||||||
docker-compose up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
**验证安装**
|
|
||||||
```shell
|
|
||||||
docker-compose ps
|
|
||||||
# 验证启动 - 状态为 UP 则表示成功
|
|
||||||
Name Command State Ports
|
|
||||||
----------------------------------------------------------------------------------------------------
|
|
||||||
elasticsearch-single /usr/local/bin/docker-entr ... Up 9200/tcp, 9300/tcp
|
|
||||||
knowstreaming-init /bin/bash /es_template_cre ... Up
|
|
||||||
knowstreaming-manager /bin/sh /ks-start.sh Up 80/tcp
|
|
||||||
knowstreaming-mysql /entrypoint.sh mysqld Up (health: starting) 3306/tcp, 33060/tcp
|
|
||||||
knowstreaming-ui /docker-entrypoint.sh ngin ... Up 0.0.0.0:80->80/tcp
|
|
||||||
|
|
||||||
# 稍等一分钟左右 knowstreaming-init 会退出，表示es初始化完成，可以访问页面
|
|
||||||
Name Command State Ports
|
|
||||||
-------------------------------------------------------------------------------------------
|
|
||||||
knowstreaming-init /bin/bash /es_template_cre ... Exit 0
|
|
||||||
knowstreaming-mysql /entrypoint.sh mysqld Up (healthy) 3306/tcp, 33060/tcp
|
|
||||||
```
|
|
||||||
|
|
||||||
**访问**
|
|
||||||
```http request
|
|
||||||
http://127.0.0.1:80/
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
**docker-compose.yml**
|
|
||||||
```yml
|
|
||||||
version: "2"
|
|
||||||
services:
|
|
||||||
# *不要调整knowstreaming-manager服务名称，ui中会用到
|
|
||||||
knowstreaming-manager:
|
|
||||||
image: knowstreaming/knowstreaming-manager:latest
|
|
||||||
container_name: knowstreaming-manager
|
|
||||||
privileged: true
|
|
||||||
restart: always
|
|
||||||
depends_on:
|
|
||||||
- elasticsearch-single
|
|
||||||
- knowstreaming-mysql
|
|
||||||
expose:
|
|
||||||
- 80
|
|
||||||
command:
|
|
||||||
- /bin/sh
|
|
||||||
- /ks-start.sh
|
|
||||||
environment:
|
|
||||||
TZ: Asia/Shanghai
|
|
||||||
# mysql服务地址
|
|
||||||
SERVER_MYSQL_ADDRESS: knowstreaming-mysql:3306
|
|
||||||
# mysql数据库名
|
|
||||||
SERVER_MYSQL_DB: know_streaming
|
|
||||||
# mysql用户名
|
|
||||||
SERVER_MYSQL_USER: root
|
|
||||||
# mysql用户密码
|
|
||||||
SERVER_MYSQL_PASSWORD: admin2022_
|
|
||||||
# es服务地址
|
|
||||||
SERVER_ES_ADDRESS: elasticsearch-single:9200
|
|
||||||
# 服务JVM参数
|
|
||||||
JAVA_OPTS: -Xmx1g -Xms1g
|
|
||||||
# 对于kafka中ADVERTISED_LISTENERS填写的hostname可以通过该方式完成
|
|
||||||
# extra_hosts:
|
|
||||||
# - "hostname:x.x.x.x"
|
|
||||||
# 服务日志路径
|
|
||||||
# volumes:
|
|
||||||
# - /ks/manage/log:/logs
|
|
||||||
knowstreaming-ui:
|
|
||||||
image: knowstreaming/knowstreaming-ui:latest
|
|
||||||
container_name: knowstreaming-ui
|
|
||||||
restart: always
|
|
||||||
ports:
|
|
||||||
- '80:80'
|
|
||||||
environment:
|
|
||||||
TZ: Asia/Shanghai
|
|
||||||
depends_on:
|
|
||||||
- knowstreaming-manager
|
|
||||||
# extra_hosts:
|
|
||||||
# - "hostname:x.x.x.x"
|
|
||||||
elasticsearch-single:
|
|
||||||
image: docker.io/library/elasticsearch:7.6.2
|
|
||||||
container_name: elasticsearch-single
|
|
||||||
restart: always
|
|
||||||
expose:
|
|
||||||
- 9200
|
|
||||||
- 9300
|
|
||||||
# ports:
|
|
||||||
# - '9200:9200'
|
|
||||||
# - '9300:9300'
|
|
||||||
environment:
|
|
||||||
TZ: Asia/Shanghai
|
|
||||||
# es的JVM参数
|
|
||||||
ES_JAVA_OPTS: -Xms512m -Xmx512m
|
|
||||||
# 单节点配置，多节点集群参考 https://www.elastic.co/guide/en/elasticsearch/reference/7.6/docker.html#docker-compose-file
|
|
||||||
discovery.type: single-node
|
|
||||||
# 数据持久化路径
|
|
||||||
# volumes:
|
|
||||||
# - /ks/es/data:/usr/share/elasticsearch/data
|
|
||||||
|
|
||||||
# es初始化服务，与manager使用同一镜像
|
|
||||||
# 首次启动es需初始化模版和索引,后续会自动创建
|
|
||||||
knowstreaming-init:
|
|
||||||
image: knowstreaming/knowstreaming-manager:latest
|
|
||||||
container_name: knowstreaming-init
|
|
||||||
depends_on:
|
|
||||||
- elasticsearch-single
|
|
||||||
command:
|
|
||||||
- /bin/bash
|
|
||||||
- /es_template_create.sh
|
|
||||||
environment:
|
|
||||||
TZ: Asia/Shanghai
|
|
||||||
# es服务地址
|
|
||||||
SERVER_ES_ADDRESS: elasticsearch-single:9200
|
|
||||||
|
|
||||||
knowstreaming-mysql:
|
|
||||||
image: knowstreaming/knowstreaming-mysql:latest
|
|
||||||
container_name: knowstreaming-mysql
|
|
||||||
restart: always
|
|
||||||
environment:
|
|
||||||
TZ: Asia/Shanghai
|
|
||||||
# root 用户密码
|
|
||||||
MYSQL_ROOT_PASSWORD: admin2022_
|
|
||||||
# 初始化时创建的数据库名称
|
|
||||||
MYSQL_DATABASE: know_streaming
|
|
||||||
# 通配所有host,可以访问远程
|
|
||||||
MYSQL_ROOT_HOST: '%'
|
|
||||||
expose:
|
|
||||||
- 3306
|
|
||||||
# ports:
|
|
||||||
# - '3306:3306'
|
|
||||||
# 数据持久化路径
|
|
||||||
# volumes:
|
|
||||||
# - /ks/mysql/data:/data/mysql
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 2.1.4、手动部署
|
|
||||||
|
|
||||||
**部署流程**
|
|
||||||
|
|
||||||
1. 安装 `JDK-11`、`MySQL`、`ElasticSearch` 等依赖服务
|
|
||||||
2. 安装 KnowStreaming
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 2.1.4.1、安装 MySQL 服务
|
|
||||||
|
|
||||||
**yum 方式安装**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 配置yum源
|
|
||||||
wget https://dev.mysql.com/get/mysql57-community-release-el7-9.noarch.rpm
|
|
||||||
rpm -ivh mysql57-community-release-el7-9.noarch.rpm
|
|
||||||
|
|
||||||
# 执行安装
|
|
||||||
yum -y install mysql-server mysql-client
|
|
||||||
|
|
||||||
# 服务启动
|
|
||||||
systemctl start mysqld
|
|
||||||
|
|
||||||
# 获取初始密码并修改
|
|
||||||
old_pass=`grep 'temporary password' /var/log/mysqld.log | awk '{print $NF}' | tail -n 1`
|
|
||||||
|
|
||||||
mysql -NBe "alter user USER() identified by 'Didi_km_678';" --connect-expired-password -uroot -p$old_pass
|
|
||||||
```
|
|
||||||
|
|
||||||
**rpm 方式安装**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 下载安装包
|
|
||||||
wget https://s3-gzpu.didistatic.com/knowsearch/mysql5.7.tar.gz
|
|
||||||
|
|
||||||
# 解压到指定目录
|
|
||||||
tar -zxf mysql5.7.tar.gz -C /tmp/
|
|
||||||
|
|
||||||
# 执行安装
|
|
||||||
yum -y localinstall /tmp/libaio-*.rpm /tmp/mysql-*.rpm
|
|
||||||
|
|
||||||
# 服务启动
|
|
||||||
systemctl start mysqld
|
|
||||||
|
|
||||||
|
|
||||||
# 获取初始密码并修改
|
|
||||||
old_pass=`grep 'temporary password' /var/log/mysqld.log | awk '{print $NF}' | tail -n 1`
|
|
||||||
|
|
||||||
mysql -NBe "alter user USER() identified by 'Didi_km_678';" --connect-expired-password -uroot -p$old_pass
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 2.1.4.2、配置 JDK 环境
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 下载安装包
|
|
||||||
wget https://s3-gzpu.didistatic.com/pub/jdk11.tar.gz
|
|
||||||
|
|
||||||
# 解压到指定目录
|
|
||||||
tar -zxf jdk11.tar.gz -C /usr/local/
|
|
||||||
|
|
||||||
# 更改目录名
|
|
||||||
mv /usr/local/jdk-11.0.2 /usr/local/java11
|
|
||||||
|
|
||||||
# 添加到环境变量
|
|
||||||
echo "export JAVA_HOME=/usr/local/java11" >> ~/.bashrc
|
|
||||||
echo "export CLASSPATH=/usr/java/java11/lib" >> ~/.bashrc
|
|
||||||
echo "export PATH=$JAVA_HOME/bin:$PATH:$HOME/bin" >> ~/.bashrc
|
|
||||||
|
|
||||||
source ~/.bashrc
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 2.1.4.3、ElasticSearch 实例搭建
|
|
||||||
|
|
||||||
- ElasticSearch 用于存储平台采集的 Kafka 指标；
|
|
||||||
- 以下安装示例为单节点模式，如需集群部署可以参考：[Elasticsearch 官方文档](https://www.elastic.co/guide/en/elasticsearch/reference/7.6/elasticsearch-intro.html)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 下载安装包
|
|
||||||
wget https://s3-gzpu.didistatic.com/pub/elasticsearch.tar.gz
|
|
||||||
|
|
||||||
# 创建ES数据存储目录
|
|
||||||
mkdir -p /data/es_data
|
|
||||||
|
|
||||||
# 创建ES所属用户
|
|
||||||
useradd arius
|
|
||||||
|
|
||||||
# 配置用户的打开文件数
|
|
||||||
echo "arius soft nofile 655350" >> /etc/security/limits.conf
|
|
||||||
echo "arius hard nofile 655350" >> /etc/security/limits.conf
|
|
||||||
echo "vm.max_map_count = 655360" >> /etc/sysctl.conf
|
|
||||||
sysctl -p
|
|
||||||
|
|
||||||
# 解压安装包
|
|
||||||
tar -zxf elasticsearch.tar.gz -C /data/
|
|
||||||
|
|
||||||
# 更改目录所属组
|
|
||||||
chown -R arius:arius /data/
|
|
||||||
|
|
||||||
# 修改配置文件(参考以下配置)
|
|
||||||
vim /data/elasticsearch/config/elasticsearch.yml
|
|
||||||
cluster.name: km_es
|
|
||||||
node.name: es-node1
|
|
||||||
node.master: true
|
|
||||||
node.data: true
|
|
||||||
path.data: /data/es_data
|
|
||||||
http.port: 8060
|
|
||||||
discovery.seed_hosts: ["127.0.0.1:9300"]
|
|
||||||
|
|
||||||
# 修改内存配置
|
|
||||||
vim /data/elasticsearch/config/jvm.options
|
|
||||||
-Xms2g
|
|
||||||
-Xmx2g
|
|
||||||
|
|
||||||
# 启动服务
|
|
||||||
su - arius
|
|
||||||
export JAVA_HOME=/usr/local/java11
|
|
||||||
sh /data/elasticsearch/control.sh start
|
|
||||||
|
|
||||||
# 确认状态
|
|
||||||
sh /data/elasticsearch/control.sh status
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 2.1.4.4、KnowStreaming 实例搭建
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 下载安装包
|
|
||||||
wget https://s3-gzpu.didistatic.com/pub/knowstreaming/KnowStreaming-3.0.0-beta.1.tar.gz
|
|
||||||
|
|
||||||
# 解压安装包到指定目录
|
|
||||||
tar -zxf KnowStreaming-3.0.0-beta.1.tar.gz -C /data/
|
|
||||||
|
|
||||||
# 修改启动脚本并加入systemd管理
|
|
||||||
cd /data/KnowStreaming/
|
|
||||||
|
|
||||||
# 创建相应的库和导入初始化数据
|
|
||||||
mysql -uroot -pDidi_km_678 -e "create database know_streaming;"
|
|
||||||
mysql -uroot -pDidi_km_678 know_streaming < ./init/sql/ddl-ks-km.sql
|
|
||||||
mysql -uroot -pDidi_km_678 know_streaming < ./init/sql/ddl-logi-job.sql
|
|
||||||
mysql -uroot -pDidi_km_678 know_streaming < ./init/sql/ddl-logi-security.sql
|
|
||||||
mysql -uroot -pDidi_km_678 know_streaming < ./init/sql/dml-ks-km.sql
|
|
||||||
mysql -uroot -pDidi_km_678 know_streaming < ./init/sql/dml-logi.sql
|
|
||||||
|
|
||||||
# 创建elasticsearch初始化数据
|
|
||||||
sh ./bin/init_es_template.sh
|
|
||||||
|
|
||||||
# 修改配置文件
|
|
||||||
vim ./conf/application.yml
|
|
||||||
|
|
||||||
# 监听端口
|
|
||||||
server:
|
|
||||||
port: 8080 # web 服务端口
|
|
||||||
tomcat:
|
|
||||||
accept-count: 1000
|
|
||||||
max-connections: 10000
|
|
||||||
|
|
||||||
# ES地址
|
|
||||||
es.client.address: 127.0.0.1:8060
|
|
||||||
|
|
||||||
# 数据库配置（一共三处地方，修改正确的mysql地址和数据库名称以及用户名密码）
|
|
||||||
jdbc-url: jdbc:mariadb://127.0.0.1:3306/know_streaming?.....
|
|
||||||
username: root
|
|
||||||
password: Didi_km_678
|
|
||||||
|
|
||||||
# 启动服务
|
|
||||||
cd /data/KnowStreaming/bin/
|
|
||||||
sh startup.sh
|
|
||||||
```
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
# `Know Streaming` 源码编译打包手册
|
|
||||||
|
|
||||||
## 1、环境信息
|
|
||||||
|
|
||||||
**系统支持**
|
|
||||||
|
|
||||||
`windows7+`、`Linux`、`Mac`
|
|
||||||
|
|
||||||
**环境依赖**
|
|
||||||
|
|
||||||
- Maven 3.6.3 （后端）
|
|
||||||
- Node v12.20.0/v14.17.3 （前端）
|
|
||||||
- Java 8+ （后端）
|
|
||||||
- Git
|
|
||||||
|
|
||||||
## 2、编译打包
|
|
||||||
|
|
||||||
整个工程中，除了`km-console`为前端模块之外，其他模块都是后端工程相关模块。
|
|
||||||
|
|
||||||
因此，如果前后端合并打包，则打对整个工程进行打包；如果前端单独打包，则仅打包 `km-console` 中的代码；如果是仅需要后端打包，则在顶层 `pom.xml` 中去掉 `km-console`模块，然后进行打包。
|
|
||||||
|
|
||||||
具体见下面描述。
|
|
||||||
|
|
||||||
### 2.1、前后端合并打包
|
|
||||||
|
|
||||||
1. 下载源码；
|
|
||||||
2. 进入 `KS-KM` 工程目录，执行 `mvn -Prelease-package -Dmaven.test.skip=true clean install -U` 命令；
|
|
||||||
3. 打包命令执行完成后，会在 `km-dist/target` 目录下面生成一个 `KnowStreaming-*.tar.gz` 的安装包。
|
|
||||||
|
|
||||||
### 2.2、前端单独打包
|
|
||||||
|
|
||||||
1. 下载源码；
|
|
||||||
2. 跳转到 [前端打包构建文档](https://github.com/didi/KnowStreaming/blob/master/km-console/README.md) 按步骤进行。打包成功后，会在 `km-rest/src/main/resources` 目录下生成名为 `templates` 的前端静态资源包;
|
|
||||||
3. 如果上一步过程中报错，请查看 [FAQ](https://github.com/didi/KnowStreaming/blob/master/docs/user_guide/faq.md) 第 8.10 条;
|
|
||||||
|
|
||||||
### 2.3、后端单独打包
|
|
||||||
|
|
||||||
1. 下载源码；
|
|
||||||
2. 修改顶层 `pom.xml` ，去掉其中的 `km-console` 模块，如下所示；
|
|
||||||
|
|
||||||
```xml
|
|
||||||
<modules>
|
|
||||||
<!-- <module>km-console</module>-->
|
|
||||||
<module>km-common</module>
|
|
||||||
<module>km-persistence</module>
|
|
||||||
<module>km-core</module>
|
|
||||||
<module>km-biz</module>
|
|
||||||
<module>km-extends/km-account</module>
|
|
||||||
<module>km-extends/km-monitor</module>
|
|
||||||
<module>km-extends/km-license</module>
|
|
||||||
<module>km-extends/km-rebalance</module>
|
|
||||||
<module>km-task</module>
|
|
||||||
<module>km-collector</module>
|
|
||||||
<module>km-rest</module>
|
|
||||||
<module>km-dist</module>
|
|
||||||
</modules>
|
|
||||||
```
|
|
||||||
|
|
||||||
3. 执行 `mvn -U clean package -Dmaven.test.skip=true`命令；
|
|
||||||
4. 执行完成之后会在 `KS-KM/km-rest/target` 目录下面生成一个 `ks-km.jar` 即为 KS 的后端部署的 Jar 包，也可以执行 `mvn -Prelease-package -Dmaven.test.skip=true clean install -U` 生成的 tar 包也仅有后端服务的功能；
|
|
||||||
@@ -1,526 +0,0 @@
|
|||||||
## 6.2、版本升级手册
|
|
||||||
|
|
||||||
注意：
|
|
||||||
- 如果想升级至具体版本，需要将你当前版本至你期望使用版本的变更统统执行一遍，然后才能正常使用。
|
|
||||||
- 如果中间某个版本没有升级信息，则表示该版本直接替换安装包即可从前一个版本升级至当前版本。
|
|
||||||
|
|
||||||
### 升级至 `master` 版本
|
|
||||||
|
|
||||||
暂无
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 升级至 `3.4.0` 版本
|
|
||||||
|
|
||||||
**配置变更**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# 新增的配置
|
|
||||||
request: # 请求相关的配置
|
|
||||||
api-call: # api调用
|
|
||||||
timeout-unit-ms: 8000 # 超时时间，默认8000毫秒
|
|
||||||
```
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
```sql
|
|
||||||
-- 多集群管理权限2023-06-27新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2026', 'Connector-新增', '1593', '1', '2', 'Connector-新增', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2028', 'Connector-编辑', '1593', '1', '2', 'Connector-编辑', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2030', 'Connector-删除', '1593', '1', '2', 'Connector-删除', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2032', 'Connector-重启', '1593', '1', '2', 'Connector-重启', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2034', 'Connector-暂停&恢复', '1593', '1', '2', 'Connector-暂停&恢复', '0', 'know-streaming');
|
|
||||||
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2026', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2028', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2030', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2032', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2034', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
-- 多集群管理权限2023-06-29新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2036', 'Security-ACL新增', '1593', '1', '2', 'Security-ACL新增', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2038', 'Security-ACL删除', '1593', '1', '2', 'Security-ACL删除', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2040', 'Security-User新增', '1593', '1', '2', 'Security-User新增', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2042', 'Security-User删除', '1593', '1', '2', 'Security-User删除', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2044', 'Security-User修改密码', '1593', '1', '2', 'Security-User修改密码', '0', 'know-streaming');
|
|
||||||
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2036', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2038', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2040', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2042', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2044', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
-- 多集群管理权限2023-07-06新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2046', 'Group-删除', '1593', '1', '2', 'Group-删除', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2048', 'GroupOffset-Topic纬度删除', '1593', '1', '2', 'GroupOffset-Topic纬度删除', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2050', 'GroupOffset-Partition纬度删除', '1593', '1', '2', 'GroupOffset-Partition纬度删除', '0', 'know-streaming');
|
|
||||||
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2046', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2048', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2050', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
-- 多集群管理权限2023-07-18新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2052', 'Security-User查看密码', '1593', '1', '2', 'Security-User查看密码', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2052', '0', 'know-streaming');
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 升级至 `3.3.0` 版本
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
```sql
|
|
||||||
ALTER TABLE `logi_security_user`
|
|
||||||
CHANGE COLUMN `phone` `phone` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'mobile' ;
|
|
||||||
|
|
||||||
ALTER TABLE ks_kc_connector ADD `heartbeat_connector_name` varchar(512) DEFAULT '' COMMENT '心跳检测connector名称';
|
|
||||||
ALTER TABLE ks_kc_connector ADD `checkpoint_connector_name` varchar(512) DEFAULT '' COMMENT '进度确认connector名称';
|
|
||||||
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_TOTAL_RECORD_ERRORS', '{\"value\" : 1}', 'MirrorMaker消息处理错误的次数', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_REPLICATION_LATENCY_MS_MAX', '{\"value\" : 6000}', 'MirrorMaker消息复制最大延迟时间', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_UNASSIGNED_TASK_COUNT', '{\"value\" : 20}', 'MirrorMaker未被分配的任务数量', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_MIRROR_MAKER_FAILED_TASK_COUNT', '{\"value\" : 10}', 'MirrorMaker失败状态的任务数量', 'admin');
|
|
||||||
|
|
||||||
|
|
||||||
-- 多集群管理权限2023-01-05新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2012', 'Topic-新增Topic复制', '1593', '1', '2', 'Topic-新增Topic复制', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2014', 'Topic-详情-取消Topic复制', '1593', '1', '2', 'Topic-详情-取消Topic复制', '0', 'know-streaming');
|
|
||||||
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2012', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2014', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
-- 多集群管理权限2023-01-18新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2016', 'MM2-新增', '1593', '1', '2', 'MM2-新增', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2018', 'MM2-编辑', '1593', '1', '2', 'MM2-编辑', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2020', 'MM2-删除', '1593', '1', '2', 'MM2-删除', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2022', 'MM2-重启', '1593', '1', '2', 'MM2-重启', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2024', 'MM2-暂停&恢复', '1593', '1', '2', 'MM2-暂停&恢复', '0', 'know-streaming');
|
|
||||||
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2016', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2018', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2020', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2022', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2024', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
DROP TABLE IF EXISTS `ks_ha_active_standby_relation`;
|
|
||||||
CREATE TABLE `ks_ha_active_standby_relation` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
|
||||||
`active_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '主集群ID',
|
|
||||||
`standby_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '备集群ID',
|
|
||||||
`res_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '资源名称',
|
|
||||||
`res_type` int(11) NOT NULL DEFAULT '-1' COMMENT '资源类型，0：集群，1：镜像Topic，2：主备Topic',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
|
||||||
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_cluster_res` (`res_type`,`active_cluster_phy_id`,`standby_cluster_phy_id`,`res_name`),
|
|
||||||
UNIQUE KEY `uniq_res_type_standby_cluster_res_name` (`res_type`,`standby_cluster_phy_id`,`res_name`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='HA主备关系表';
|
|
||||||
|
|
||||||
|
|
||||||
-- 删除idx_cluster_phy_id 索引并新增idx_cluster_update_time索引
|
|
||||||
ALTER TABLE `ks_km_kafka_change_record` DROP INDEX `idx_cluster_phy_id` ,
|
|
||||||
ADD INDEX `idx_cluster_update_time` (`cluster_phy_id` ASC, `update_time` ASC);
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 升级至 `3.2.0` 版本
|
|
||||||
|
|
||||||
**配置变更**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# 新增如下配置
|
|
||||||
|
|
||||||
spring:
|
|
||||||
logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
|
|
||||||
enable: true # true表示开启job任务, false表关闭。KS在部署上可以考虑部署两套服务，一套处理前端请求，一套执行job任务，此时可以通过该字段进行控制
|
|
||||||
|
|
||||||
# 线程池大小相关配置
|
|
||||||
thread-pool:
|
|
||||||
es:
|
|
||||||
search: # es查询线程池
|
|
||||||
thread-num: 20 # 线程池大小
|
|
||||||
queue-size: 10000 # 队列大小
|
|
||||||
|
|
||||||
# 客户端池大小相关配置
|
|
||||||
client-pool:
|
|
||||||
kafka-admin:
|
|
||||||
client-cnt: 1 # 每个Kafka集群创建的KafkaAdminClient数
|
|
||||||
|
|
||||||
# ES客户端配置
|
|
||||||
es:
|
|
||||||
index:
|
|
||||||
expire: 15 # 索引过期天数，15表示超过15天的索引会被KS过期删除
|
|
||||||
```
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
```sql
|
|
||||||
DROP TABLE IF EXISTS `ks_kc_connect_cluster`;
|
|
||||||
CREATE TABLE `ks_kc_connect_cluster` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Connect集群ID',
|
|
||||||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
|
||||||
`name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群名称',
|
|
||||||
`group_name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群Group名称',
|
|
||||||
`cluster_url` varchar(1024) NOT NULL DEFAULT '' COMMENT '集群地址',
|
|
||||||
`member_leader_url` varchar(1024) NOT NULL DEFAULT '' COMMENT 'URL地址',
|
|
||||||
`version` varchar(64) NOT NULL DEFAULT '' COMMENT 'connect版本',
|
|
||||||
`jmx_properties` text COMMENT 'JMX配置',
|
|
||||||
`state` tinyint(4) NOT NULL DEFAULT '1' COMMENT '集群使用的消费组状态，也表示集群状态:-1 Unknown,0 ReBalance,1 Active,2 Dead,3 Empty',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '接入时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_id_group_name` (`id`,`group_name`),
|
|
||||||
UNIQUE KEY `uniq_name_kafka_cluster` (`name`,`kafka_cluster_phy_id`),
|
|
||||||
KEY `idx_kafka_cluster_phy_id` (`kafka_cluster_phy_id`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Connect集群信息表';
|
|
||||||
|
|
||||||
|
|
||||||
DROP TABLE IF EXISTS `ks_kc_connector`;
|
|
||||||
CREATE TABLE `ks_kc_connector` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
|
||||||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
|
||||||
`connect_cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Connect集群ID',
|
|
||||||
`connector_name` varchar(512) NOT NULL DEFAULT '' COMMENT 'Connector名称',
|
|
||||||
`connector_class_name` varchar(512) NOT NULL DEFAULT '' COMMENT 'Connector类',
|
|
||||||
`connector_type` varchar(32) NOT NULL DEFAULT '' COMMENT 'Connector类型',
|
|
||||||
`state` varchar(45) NOT NULL DEFAULT '' COMMENT '状态',
|
|
||||||
`topics` text COMMENT '访问过的Topics',
|
|
||||||
`task_count` int(11) NOT NULL DEFAULT '0' COMMENT '任务数',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_connect_cluster_id_connector_name` (`connect_cluster_id`,`connector_name`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Connector信息表';
|
|
||||||
|
|
||||||
|
|
||||||
DROP TABLE IF EXISTS `ks_kc_worker`;
|
|
||||||
CREATE TABLE `ks_kc_worker` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
|
||||||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
|
||||||
`connect_cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Connect集群ID',
|
|
||||||
`member_id` varchar(512) NOT NULL DEFAULT '' COMMENT '成员ID',
|
|
||||||
`host` varchar(128) NOT NULL DEFAULT '' COMMENT '主机名',
|
|
||||||
`jmx_port` int(16) NOT NULL DEFAULT '-1' COMMENT 'Jmx端口',
|
|
||||||
`url` varchar(1024) NOT NULL DEFAULT '' COMMENT 'URL信息',
|
|
||||||
`leader_url` varchar(1024) NOT NULL DEFAULT '' COMMENT 'leaderURL信息',
|
|
||||||
`leader` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1是leader，0不是leader',
|
|
||||||
`worker_id` varchar(128) NOT NULL COMMENT 'worker地址',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_cluster_id_member_id` (`connect_cluster_id`,`member_id`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker信息表';
|
|
||||||
|
|
||||||
|
|
||||||
DROP TABLE IF EXISTS `ks_kc_worker_connector`;
|
|
||||||
CREATE TABLE `ks_kc_worker_connector` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
|
||||||
`kafka_cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Kafka集群ID',
|
|
||||||
`connect_cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'Connect集群ID',
|
|
||||||
`connector_name` varchar(512) NOT NULL DEFAULT '' COMMENT 'Connector名称',
|
|
||||||
`worker_member_id` varchar(256) NOT NULL DEFAULT '',
|
|
||||||
`task_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'Task的ID',
|
|
||||||
`state` varchar(128) DEFAULT NULL COMMENT '任务状态',
|
|
||||||
`worker_id` varchar(128) DEFAULT NULL COMMENT 'worker信息',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_relation` (`connect_cluster_id`,`connector_name`,`task_id`,`worker_member_id`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Worker和Connector关系表';
|
|
||||||
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_CONNECTOR_FAILED_TASK_COUNT', '{\"value\" : 1}', 'connector失败状态的任务数量', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_CONNECTOR_UNASSIGNED_TASK_COUNT', '{\"value\" : 1}', 'connector未被分配的任务数量', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_CONNECT_CLUSTER_TASK_STARTUP_FAILURE_PERCENTAGE', '{\"value\" : 0.05}', 'Connect集群任务启动失败概率', 'admin');
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 升级至 `v3.1.0` 版本
|
|
||||||
|
|
||||||
```sql
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_BRAIN_SPLIT', '{ \"value\": 1} ', 'ZK 脑裂', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_OUTSTANDING_REQUESTS', '{ \"amount\": 100, \"ratio\":0.8} ', 'ZK Outstanding 请求堆积数', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_WATCH_COUNT', '{ \"amount\": 100000, \"ratio\": 0.8 } ', 'ZK WatchCount 数', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_ALIVE_CONNECTIONS', '{ \"amount\": 10000, \"ratio\": 0.8 } ', 'ZK 连接数', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_APPROXIMATE_DATA_SIZE', '{ \"amount\": 524288000, \"ratio\": 0.8 } ', 'ZK 数据大小(Byte)', 'admin');
|
|
||||||
INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`, `value_group`, `value_name`, `value`, `description`, `operator`) VALUES ('-1', 'HEALTH', 'HC_ZK_SENT_RATE', '{ \"amount\": 500000, \"ratio\": 0.8 } ', 'ZK 发包数', 'admin');
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
### 升级至 `v3.0.1` 版本
|
|
||||||
|
|
||||||
**ES 索引模版**
|
|
||||||
```bash
|
|
||||||
# 新增 ks_kafka_zookeeper_metric 索引模版。
|
|
||||||
# 可通过再次执行 bin/init_es_template.sh 脚本，创建该索引模版。
|
|
||||||
|
|
||||||
# 索引模版内容
|
|
||||||
PUT _template/ks_kafka_zookeeper_metric
|
|
||||||
{
|
|
||||||
"order" : 10,
|
|
||||||
"index_patterns" : [
|
|
||||||
"ks_kafka_zookeeper_metric*"
|
|
||||||
],
|
|
||||||
"settings" : {
|
|
||||||
"index" : {
|
|
||||||
"number_of_shards" : "10"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"mappings" : {
|
|
||||||
"properties" : {
|
|
||||||
"routingValue" : {
|
|
||||||
"type" : "text",
|
|
||||||
"fields" : {
|
|
||||||
"keyword" : {
|
|
||||||
"ignore_above" : 256,
|
|
||||||
"type" : "keyword"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"clusterPhyId" : {
|
|
||||||
"type" : "long"
|
|
||||||
},
|
|
||||||
"metrics" : {
|
|
||||||
"properties" : {
|
|
||||||
"AvgRequestLatency" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"MinRequestLatency" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"MaxRequestLatency" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"OutstandingRequests" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"NodeCount" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"WatchCount" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"NumAliveConnections" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"PacketsReceived" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"PacketsSent" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"EphemeralsCount" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"ApproximateDataSize" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"OpenFileDescriptorCount" : {
|
|
||||||
"type" : "double"
|
|
||||||
},
|
|
||||||
"MaxFileDescriptorCount" : {
|
|
||||||
"type" : "double"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"key" : {
|
|
||||||
"type" : "text",
|
|
||||||
"fields" : {
|
|
||||||
"keyword" : {
|
|
||||||
"ignore_above" : 256,
|
|
||||||
"type" : "keyword"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"timestamp" : {
|
|
||||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
|
||||||
"type" : "date"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"aliases" : { }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
DROP TABLE IF EXISTS `ks_km_zookeeper`;
|
|
||||||
CREATE TABLE `ks_km_zookeeper` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
|
||||||
`cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '物理集群ID',
|
|
||||||
`host` varchar(128) NOT NULL DEFAULT '' COMMENT 'zookeeper主机名',
|
|
||||||
`port` int(16) NOT NULL DEFAULT '-1' COMMENT 'zookeeper端口',
|
|
||||||
`role` varchar(16) NOT NULL DEFAULT '' COMMENT '角色, leader follower observer',
|
|
||||||
`version` varchar(128) NOT NULL DEFAULT '' COMMENT 'zookeeper版本',
|
|
||||||
`status` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1存活，0未存活，11存活但是4字命令使用不了',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_cluster_phy_id_host_port` (`cluster_phy_id`,`host`, `port`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Zookeeper信息表';
|
|
||||||
|
|
||||||
|
|
||||||
DROP TABLE IF EXISTS `ks_km_group`;
|
|
||||||
CREATE TABLE `ks_km_group` (
|
|
||||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
|
|
||||||
`cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群id',
|
|
||||||
`name` varchar(192) COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Group名称',
|
|
||||||
`member_count` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '成员数',
|
|
||||||
`topic_members` text CHARACTER SET utf8 COMMENT 'group消费的topic列表',
|
|
||||||
`partition_assignor` varchar(255) CHARACTER SET utf8 NOT NULL COMMENT '分配策略',
|
|
||||||
`coordinator_id` int(11) NOT NULL COMMENT 'group协调器brokerId',
|
|
||||||
`type` int(11) NOT NULL COMMENT 'group类型 0：consumer 1：connector',
|
|
||||||
`state` varchar(64) CHARACTER SET utf8 NOT NULL DEFAULT '' COMMENT '状态',
|
|
||||||
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
|
||||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
UNIQUE KEY `uniq_cluster_phy_id_name` (`cluster_phy_id`,`name`)
|
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Group信息表';
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
### 升级至 `v3.0.0` 版本
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
ALTER TABLE `ks_km_physical_cluster`
|
|
||||||
ADD COLUMN `zk_properties` TEXT NULL COMMENT 'ZK配置' AFTER `jmx_properties`;
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### 升级至 `v3.0.0-beta.2`版本
|
|
||||||
|
|
||||||
**配置变更**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
|
|
||||||
# 新增配置
|
|
||||||
spring:
|
|
||||||
logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
|
|
||||||
login-extend-bean-name: logiSecurityDefaultLoginExtendImpl # 使用的登录系统Service的Bean名称，无需修改
|
|
||||||
|
|
||||||
# 线程池大小相关配置，在task模块中，新增了三类线程池，
|
|
||||||
# 从而减少不同类型任务之间的相互影响，以及减少对logi-job内的线程池的影响
|
|
||||||
thread-pool:
|
|
||||||
task: # 任务模块的配置
|
|
||||||
metrics: # metrics采集任务配置
|
|
||||||
thread-num: 18 # metrics采集任务线程池核心线程数
|
|
||||||
queue-size: 180 # metrics采集任务线程池队列大小
|
|
||||||
metadata: # metadata同步任务配置
|
|
||||||
thread-num: 27 # metadata同步任务线程池核心线程数
|
|
||||||
queue-size: 270 # metadata同步任务线程池队列大小
|
|
||||||
common: # 剩余其他任务配置
|
|
||||||
thread-num: 15 # 剩余其他任务线程池核心线程数
|
|
||||||
queue-size: 150 # 剩余其他任务线程池队列大小
|
|
||||||
|
|
||||||
# 删除配置，下列配置将不再使用
|
|
||||||
thread-pool:
|
|
||||||
task: # 任务模块的配置
|
|
||||||
heaven: # 采集任务配置
|
|
||||||
thread-num: 20 # 采集任务线程池核心线程数
|
|
||||||
queue-size: 1000 # 采集任务线程池队列大小
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
-- 多集群管理权限2022-09-06新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2000', '多集群管理查看', '1593', '1', '2', '多集群管理查看', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2002', 'Topic-迁移副本', '1593', '1', '2', 'Topic-迁移副本', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2004', 'Topic-扩缩副本', '1593', '1', '2', 'Topic-扩缩副本', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2006', 'Cluster-LoadReBalance-周期均衡', '1593', '1', '2', 'Cluster-LoadReBalance-周期均衡', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2008', 'Cluster-LoadReBalance-立即均衡', '1593', '1', '2', 'Cluster-LoadReBalance-立即均衡', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2010', 'Cluster-LoadReBalance-设置集群规格', '1593', '1', '2', 'Cluster-LoadReBalance-设置集群规格', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
-- 系统管理权限2022-09-06新增
|
|
||||||
INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('3000', '系统管理查看', '1595', '1', '2', '系统管理查看', '0', 'know-streaming');
|
|
||||||
|
|
||||||
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2000', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2002', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2004', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2006', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2008', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2010', '0', 'know-streaming');
|
|
||||||
INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '3000', '0', 'know-streaming');
|
|
||||||
|
|
||||||
-- 修改字段长度
|
|
||||||
ALTER TABLE `logi_security_oplog`
|
|
||||||
CHANGE COLUMN `operator_ip` `operator_ip` VARCHAR(64) NOT NULL COMMENT '操作者ip' ,
|
|
||||||
CHANGE COLUMN `operator` `operator` VARCHAR(64) NULL DEFAULT NULL COMMENT '操作者账号' ,
|
|
||||||
CHANGE COLUMN `operate_page` `operate_page` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '操作页面' ,
|
|
||||||
CHANGE COLUMN `operate_type` `operate_type` VARCHAR(64) NOT NULL COMMENT '操作类型' ,
|
|
||||||
CHANGE COLUMN `target_type` `target_type` VARCHAR(64) NOT NULL COMMENT '对象分类' ,
|
|
||||||
CHANGE COLUMN `target` `target` VARCHAR(1024) NOT NULL COMMENT '操作对象' ,
|
|
||||||
CHANGE COLUMN `operation_methods` `operation_methods` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '操作方式' ;
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 升级至 `v3.0.0-beta.1`版本
|
|
||||||
|
|
||||||
**SQL 变更**
|
|
||||||
|
|
||||||
1、在`ks_km_broker`表增加了一个监听信息字段。
|
|
||||||
2、为`logi_security_oplog`表 operation_methods 字段设置默认值''。
|
|
||||||
因此需要执行下面的 sql 对数据库表进行更新。
|
|
||||||
|
|
||||||
```sql
|
|
||||||
ALTER TABLE `ks_km_broker`
|
|
||||||
ADD COLUMN `endpoint_map` VARCHAR(1024) NOT NULL DEFAULT '' COMMENT '监听信息' AFTER `update_time`;
|
|
||||||
|
|
||||||
ALTER TABLE `logi_security_oplog`
|
|
||||||
ALTER COLUMN `operation_methods` set default '';
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### `2.x`版本 升级至 `v3.0.0-beta.0`版本
|
|
||||||
|
|
||||||
**升级步骤：**
|
|
||||||
|
|
||||||
1. 依旧使用**`2.x 版本的 DB`**，在上面初始化 3.0.0 版本所需数据库表结构及数据；
|
|
||||||
2. 将 2.x 版本中的集群，在 3.0.0 版本，手动逐一接入；
|
|
||||||
3. 将 Topic 业务数据，迁移至 3.0.0 表中，详见下方 SQL；
|
|
||||||
|
|
||||||
**注意事项**
|
|
||||||
|
|
||||||
- 建议升级 3.0.0 版本过程中，保留 2.x 版本的使用，待 3.0.0 版本稳定使用后，再下线 2.x 版本；
|
|
||||||
- 3.0.0 版本仅需要`集群信息`及`Topic的描述信息`。2.x 版本的 DB 的其他数据 3.0.0 版本都不需要；
|
|
||||||
- 部署 3.0.0 版本之后，集群、Topic 等指标数据都为空，3.0.0 版本会周期进行采集，运行一段时间之后就会有该数据了，因此不会将 2.x 中的指标数据进行迁移；
|
|
||||||
|
|
||||||
**迁移数据**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
-- 迁移Topic的备注信息。
|
|
||||||
-- 需在 3.0.0 部署完成后，再执行该SQL。
|
|
||||||
-- 考虑到 2.x 版本中还存在增量数据，因此建议改SQL周期执行，是的增量数据也能被迁移至 3.0.0 版本中。
|
|
||||||
|
|
||||||
UPDATE ks_km_topic
|
|
||||||
INNER JOIN
|
|
||||||
(SELECT
|
|
||||||
topic.cluster_id AS cluster_id,
|
|
||||||
topic.topic_name AS topic_name,
|
|
||||||
topic.description AS description
|
|
||||||
FROM topic WHERE description != ''
|
|
||||||
) AS t
|
|
||||||
|
|
||||||
ON ks_km_topic.cluster_phy_id = t.cluster_id
|
|
||||||
AND ks_km_topic.topic_name = t.topic_name
|
|
||||||
AND ks_km_topic.id > 0
|
|
||||||
SET ks_km_topic.description = t.description;
|
|
||||||
```
|
|
||||||
@@ -1,321 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
# FAQ
|
|
||||||
|
|
||||||
- [FAQ](#faq)
|
|
||||||
- [1、支持哪些 Kafka 版本？](#1支持哪些-kafka-版本)
|
|
||||||
- [1、2.x 版本和 3.0 版本有什么差异？](#12x-版本和-30-版本有什么差异)
|
|
||||||
- [3、页面流量信息等无数据？](#3页面流量信息等无数据)
|
|
||||||
- [4、`Jmx`连接失败如何解决？](#4jmx连接失败如何解决)
|
|
||||||
- [5、有没有 API 文档？](#5有没有-api-文档)
|
|
||||||
- [6、删除 Topic 成功后，为何过段时间又出现了？](#6删除-topic-成功后为何过段时间又出现了)
|
|
||||||
- [7、如何在不登录的情况下，调用接口？](#7如何在不登录的情况下调用接口)
|
|
||||||
- [8、Specified key was too long; max key length is 767 bytes](#8specified-key-was-too-long-max-key-length-is-767-bytes)
|
|
||||||
- [9、出现 ESIndexNotFoundEXception 报错](#9出现-esindexnotfoundexception-报错)
|
|
||||||
- [10、km-console 打包构建失败](#10km-console-打包构建失败)
|
|
||||||
- [11、在 `km-console` 目录下执行 `npm run start` 时看不到应用构建和热加载过程？如何启动单个应用？](#11在-km-console-目录下执行-npm-run-start-时看不到应用构建和热加载过程如何启动单个应用)
|
|
||||||
- [12、权限识别失败问题](#12权限识别失败问题)
|
|
||||||
- [13、接入开启kerberos认证的kafka集群](#13接入开启kerberos认证的kafka集群)
|
|
||||||
- [14、对接Ldap的配置](#14对接ldap的配置)
|
|
||||||
- [15、测试时使用Testcontainers的说明](#15测试时使用testcontainers的说明)
|
|
||||||
- [16、JMX连接失败怎么办](#16jmx连接失败怎么办)
|
|
||||||
- [17、zk监控无数据问题](#17zk监控无数据问题)
|
|
||||||
- [18、启动失败，报NoClassDefFoundError如何解决](#18启动失败报noclassdeffounderror如何解决)
|
|
||||||
- [19、依赖ElasticSearch 8.0以上版本部署后指标信息无法正常显示如何解决]
|
|
||||||
|
|
||||||
## 1、支持哪些 Kafka 版本？
|
|
||||||
|
|
||||||
- 支持 0.10+ 的 Kafka 版本；
|
|
||||||
- 支持 ZK 及 Raft 运行模式的 Kafka 版本；
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 1、2.x 版本和 3.0 版本有什么差异？
|
|
||||||
|
|
||||||
**全新设计理念**
|
|
||||||
|
|
||||||
- 在 0 侵入、0 门槛的前提下提供直观 GUI 用于管理和观测 Apache Kafka®，帮助用户降低 Kafka CLI 操作门槛，轻松实现对原生 Kafka 集群的可管、可见、可掌控，提升 Kafka 使用体验和降低管理成本。
|
|
||||||
- 支持海量集群一键接入，无需任何改造，即可实现集群深度纳管，真正的 0 侵入、插件化系统设计，覆盖 0.10.x-3.x.x 众多 Kafka 版本无缝纳管。
|
|
||||||
|
|
||||||
**开源协议调整**
|
|
||||||
|
|
||||||
- 3.x：AGPL 3.0
|
|
||||||
- 2.x：Apache License 2.0
|
|
||||||
|
|
||||||
更多具体内容见：[新旧版本对比](https://doc.knowstreaming.com/product/9-attachment#92%E6%96%B0%E6%97%A7%E7%89%88%E6%9C%AC%E5%AF%B9%E6%AF%94)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 3、页面流量信息等无数据？
|
|
||||||
|
|
||||||
- 1、`Broker JMX`未正确开启
|
|
||||||
|
|
||||||
可以参看：[Jmx 连接配置&问题解决](https://doc.knowstreaming.com/product/9-attachment#91jmx-%E8%BF%9E%E6%8E%A5%E5%A4%B1%E8%B4%A5%E9%97%AE%E9%A2%98%E8%A7%A3%E5%86%B3)
|
|
||||||
|
|
||||||
- 2、`ES` 存在问题
|
|
||||||
|
|
||||||
建议使用`ES 7.6`版本，同时创建近 7 天的索引，具体见：[快速开始](./1-quick-start.md) 中的 ES 索引模版及索引创建。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 4、`Jmx`连接失败如何解决？
|
|
||||||
|
|
||||||
- 参看 [Jmx 连接配置&问题解决](https://doc.knowstreaming.com/product/9-attachment#91jmx-%E8%BF%9E%E6%8E%A5%E5%A4%B1%E8%B4%A5%E9%97%AE%E9%A2%98%E8%A7%A3%E5%86%B3) 说明。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 5、有没有 API 文档？
|
|
||||||
|
|
||||||
`KnowStreaming` 采用 Swagger 进行 API 说明，在启动 KnowStreaming 服务之后，就可以从下面地址看到。
|
|
||||||
|
|
||||||
Swagger-API 地址： [http://IP:PORT/swagger-ui.html#/](http://IP:PORT/swagger-ui.html#/)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 6、删除 Topic 成功后，为何过段时间又出现了？
|
|
||||||
|
|
||||||
**原因说明：**
|
|
||||||
|
|
||||||
`KnowStreaming` 会去请求 Topic 的 endoffset 信息，要获取这个信息就需要发送 metadata 请求，发送 metadata 请求的时候，如果集群允许自动创建 Topic，那么当 Topic 不存在时，就会自动将该 Topic 创建出来。
|
|
||||||
|
|
||||||
**问题解决：**
|
|
||||||
|
|
||||||
因为在 `KnowStreaming` 上，禁止 Kafka 客户端内部元信息获取这个动作非常的难做到，因此短时间内这个问题不好从 `KnowStreaming` 上解决。
|
|
||||||
|
|
||||||
当然，对于不存在的 Topic，`KnowStreaming` 是不会进行元信息请求的，因此也不用担心会莫名其妙的创建一个 Topic 出来。
|
|
||||||
|
|
||||||
但是，另外一点，对于开启允许 Topic 自动创建的集群，建议是关闭该功能，开启是非常危险的，如果关闭之后，`KnowStreaming` 也不会有这个问题。
|
|
||||||
|
|
||||||
最后这里举个开启这个配置后，非常危险的代码例子吧：
|
|
||||||
|
|
||||||
```java
|
|
||||||
for (int i= 0; i < 100000; ++i) {
|
|
||||||
// 如果是客户端类似这样写的，那么一启动，那么将创建10万个Topic出来，集群元信息瞬间爆炸，controller可能就不可服务了。
|
|
||||||
producer.send(new ProducerRecord<String, String>("know_streaming" + i,"hello logi_km"));
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 7、如何在不登录的情况下，调用接口？
|
|
||||||
|
|
||||||
步骤一：接口调用时，在 header 中，增加如下信息：
|
|
||||||
|
|
||||||
```shell
|
|
||||||
# 表示开启登录绕过
|
|
||||||
Trick-Login-Switch : on
|
|
||||||
|
|
||||||
# 登录绕过的用户, 这里可以是admin, 或者是其他的, 但是必须在系统管理->用户管理中设置了该用户。
|
|
||||||
Trick-Login-User : admin
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
步骤二：点击右上角"系统管理"，选择配置管理，在页面中添加以下键值对。
|
|
||||||
|
|
||||||
```shell
|
|
||||||
# 模块选择
|
|
||||||
SECURITY.LOGIN
|
|
||||||
|
|
||||||
# 设置的配置键，必须是这个
|
|
||||||
SECURITY.TRICK_USERS
|
|
||||||
|
|
||||||
# 设置的value，是json数组的格式，包含步骤一header中设置的用户名，例如
|
|
||||||
[ "admin", "logi"]
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
步骤三：解释说明
|
|
||||||
|
|
||||||
设置完成上面两步之后，就可以直接调用需要登录的接口了。
|
|
||||||
|
|
||||||
但是还有一点需要注意，绕过的用户仅能调用他有权限的接口，比如一个普通用户，那么他就只能调用普通的接口，不能去调用运维人员的接口。
|
|
||||||
|
|
||||||
## 8、Specified key was too long; max key length is 767 bytes
|
|
||||||
|
|
||||||
**原因：** 不同版本的 InoDB 引擎，参数‘innodb_large_prefix’默认值不同，即在 5.6 默认值为 OFF，5.7 默认值为 ON。
|
|
||||||
|
|
||||||
对于引擎为 InnoDB，innodb_large_prefix=OFF，且行格式为 Antelope 即支持 REDUNDANT 或 COMPACT 时，索引键前缀长度最大为 767 字节。innodb_large_prefix=ON，且行格式为 Barracuda 即支持 DYNAMIC 或 COMPRESSED 时，索引键前缀长度最大为 3072 字节。
|
|
||||||
|
|
||||||
**解决方案：**
|
|
||||||
|
|
||||||
- 减少 varchar 字符大小低于 767/4=191。
|
|
||||||
- 将字符集改为 latin1（一个字符=一个字节）。
|
|
||||||
- 开启‘innodb_large_prefix’，修改默认行格式‘innodb_file_format’为 Barracuda，并设置 row_format=dynamic。
|
|
||||||
|
|
||||||
## 9、出现 ESIndexNotFoundEXception 报错
|
|
||||||
|
|
||||||
**原因 ：**没有创建 ES 索引模版
|
|
||||||
|
|
||||||
**解决方案：**执行 init_es_template.sh 脚本，创建 ES 索引模版即可。
|
|
||||||
|
|
||||||
## 10、km-console 打包构建失败
|
|
||||||
|
|
||||||
首先，**请确保您正在使用最新版本**，版本列表见 [Tags](https://github.com/didi/KnowStreaming/tags)。如果不是最新版本，请升级后再尝试有无问题。
|
|
||||||
|
|
||||||
常见的原因是由于工程依赖没有正常安装，导致在打包过程中缺少依赖，造成打包失败。您可以检查是否有以下文件夹，且文件夹内是否有内容
|
|
||||||
|
|
||||||
```
|
|
||||||
KnowStreaming/km-console/node_modules
|
|
||||||
KnowStreaming/km-console/packages/layout-clusters-fe/node_modules
|
|
||||||
KnowStreaming/km-console/packages/config-manager-fe/node_modules
|
|
||||||
```
|
|
||||||
|
|
||||||
如果发现没有对应的 `node_modules` 目录或着目录内容为空，说明依赖没有安装成功。请按以下步骤操作，
|
|
||||||
|
|
||||||
1. 手动删除上述三个文件夹（如果有）
|
|
||||||
|
|
||||||
2. 如果之前是通过 `mvn install` 打包 `km-console`，请到项目根目录（KnowStreaming）下重新输入该指令进行打包。观察打包过程有无报错。如有报错，请见步骤 4。
|
|
||||||
|
|
||||||
3. 如果是通过本地独立构建前端工程的方式（指直接执行 `npm run build`），请进入 `KnowStreaming/km-console` 目录，执行下述步骤（注意：执行时请确保您在使用 `node v12` 版本）
|
|
||||||
|
|
||||||
a. 执行 `npm run i`。如有报错，请见步骤 4。
|
|
||||||
|
|
||||||
b. 执行 `npm run build`。如有报错，请见步骤 4。
|
|
||||||
|
|
||||||
4. 麻烦联系我们协助解决。推荐提供以下信息，方面我们快速定位问题，示例如下。
|
|
||||||
|
|
||||||
```
|
|
||||||
操作系统: Mac
|
|
||||||
命令行终端：bash
|
|
||||||
Node 版本: v12.22.12
|
|
||||||
复现步骤: 1. -> 2.
|
|
||||||
错误截图:
|
|
||||||
```
|
|
||||||
|
|
||||||
## 11、在 `km-console` 目录下执行 `npm run start` 时看不到应用构建和热加载过程？如何启动单个应用？
|
|
||||||
|
|
||||||
需要到具体的应用中执行 `npm run start`，例如 `cd packages/layout-clusters-fe` 后，执行 `npm run start`。
|
|
||||||
|
|
||||||
应用启动后需要到基座应用中查看（需要启动基座应用，即 layout-clusters-fe）。
|
|
||||||
|
|
||||||
|
|
||||||
## 12、权限识别失败问题
|
|
||||||
1、使用admin账号登陆KnowStreaming时，点击系统管理-用户管理-角色管理-新增角色，查看页面是否正常。
|
|
||||||
|
|
||||||
<img src="http://img-ys011.didistatic.com/static/dc2img/do1_gwGfjN9N92UxzHU8dfzr" width = "400" >
|
|
||||||
|
|
||||||
2、查看'/logi-security/api/v1/permission/tree'接口返回值，出现如下图所示乱码现象。
|
|
||||||
|
|
||||||
|
|
||||||
3、查看logi_security_permission表，看看是否出现了中文乱码现象。
|
|
||||||
|
|
||||||
根据以上几点，我们可以确定是由于数据库乱码造成的权限识别失败问题。
|
|
||||||
|
|
||||||
+ 原因：由于数据库编码和我们提供的脚本不一致，数据库里的数据发生了乱码，因此出现权限识别失败问题。
|
|
||||||
+ 解决方案：清空数据库数据，将数据库字符集调整为utf8，最后重新执行[dml-logi.sql](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/sql/dml-logi.sql)脚本导入数据即可。
|
|
||||||
|
|
||||||
|
|
||||||
## 13、接入开启kerberos认证的kafka集群
|
|
||||||
|
|
||||||
1. 部署KnowStreaming的机器上安装krb客户端；
|
|
||||||
2. 替换/etc/krb5.conf配置文件；
|
|
||||||
3. 把kafka对应的keytab复制到改机器目录下；
|
|
||||||
4. 接入集群时认证配置，配置信息根据实际情况填写；
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"security.protocol": "SASL_PLAINTEXT",
|
|
||||||
"sasl.mechanism": "GSSAPI",
|
|
||||||
"sasl.jaas.config": "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\"/etc/keytab/kafka.keytab\" storeKey=true useTicketCache=false principal=\"kafka/kafka@TEST.COM\";",
|
|
||||||
"sasl.kerberos.service.name": "kafka"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
## 14、对接Ldap的配置
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# 需要在application.yml中增加如下配置。相关配置的信息，按实际情况进行调整
|
|
||||||
account:
|
|
||||||
ldap:
|
|
||||||
url: ldap://127.0.0.1:8080/
|
|
||||||
basedn: DC=senz,DC=local
|
|
||||||
factory: com.sun.jndi.ldap.LdapCtxFactory
|
|
||||||
filter: sAMAccountName
|
|
||||||
security:
|
|
||||||
authentication: simple
|
|
||||||
principal: CN=search,DC=senz,DC=local
|
|
||||||
credentials: xxxxxxx
|
|
||||||
auth-user-registration: false # 是否注册到mysql，默认false
|
|
||||||
auth-user-registration-role: 1677 # 1677是超级管理员角色的id，如果赋予想默认赋予普通角色，可以到ks新建一个。
|
|
||||||
|
|
||||||
# 需要在application.yml中修改如下配置
|
|
||||||
spring:
|
|
||||||
logi-security:
|
|
||||||
login-extend-bean-name: ksLdapLoginService # 表示使用ldap的service
|
|
||||||
```
|
|
||||||
|
|
||||||
## 15、测试时使用Testcontainers的说明
|
|
||||||
|
|
||||||
1. 需要docker运行环境 [Testcontainers运行环境说明](https://www.testcontainers.org/supported_docker_environment/)
|
|
||||||
2. 如果本机没有docker，可以使用[远程访问docker](https://docs.docker.com/config/daemon/remote-access/) [Testcontainers配置说明](https://www.testcontainers.org/features/configuration/#customizing-docker-host-detection)
|
|
||||||
|
|
||||||
|
|
||||||
## 16、JMX连接失败怎么办
|
|
||||||
|
|
||||||
详细见：[解决连接JMX失败](../dev_guide/%E8%A7%A3%E5%86%B3%E8%BF%9E%E6%8E%A5JMX%E5%A4%B1%E8%B4%A5.md)
|
|
||||||
|
|
||||||
|
|
||||||
## 17、zk监控无数据问题
|
|
||||||
|
|
||||||
**现象：**
|
|
||||||
zookeeper集群正常，但Ks上zk页面所有监控指标无数据，`KnowStreaming` log_error.log日志提示
|
|
||||||
|
|
||||||
```vim
|
|
||||||
[MetricCollect-Shard-0-8-thread-1] ERROR class=c.x.k.s.k.c.s.h.c.z.HealthCheckZookeeperService||method=checkWatchCount||param=ZookeeperParam(zkAddressList=[Tuple{v1=192.168.xxx.xx, v2=2181}, Tuple{v1=192.168.xxx.xx, v2=2181}, Tuple{v1=192.168.xxx.xx, v2=2181}], zkConfig=null)||config=HealthAmountRatioConfig(amount=100000, ratio=0.8)||result=Result{message='mntr is not executed because it is not in the whitelist.
|
|
||||||
', code=8031, data=null}||errMsg=get metrics failed, may be collect failed or zk mntr command not in whitelist.
|
|
||||||
2023-04-23 14:39:07.234 [MetricCollect-Shard-0-8-thread-1] ERROR class=c.x.k.s.k.c.s.h.checker.AbstractHeal
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
原因就很明确了。需要开放zk的四字命令，在`zoo.cfg`配置文件中添加
|
|
||||||
```
|
|
||||||
4lw.commands.whitelist=mntr,stat,ruok,envi,srvr,envi,cons,conf,wchs,wchp
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
建议至少开放上述几个四字命令，当然，您也可以全部开放
|
|
||||||
```
|
|
||||||
4lw.commands.whitelist=*
|
|
||||||
```
|
|
||||||
|
|
||||||
## 18、启动失败，报NoClassDefFoundError如何解决
|
|
||||||
|
|
||||||
**错误现象：**
|
|
||||||
```log
|
|
||||||
# 启动失败，报nested exception is java.lang.NoClassDefFoundError: Could not initialize class com.didiglobal.logi.job.core.WorkerSingleton$Singleton
|
|
||||||
|
|
||||||
|
|
||||||
2023-08-11 22:54:29.842 [main] ERROR class=org.springframework.boot.SpringApplication||Application run failed
|
|
||||||
org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'quartzScheduler' defined in class path resource [com/didiglobal/logi/job/LogIJobAutoConfiguration.class]: Bean instantiation via factory method failed; nested exception is org.springframework.beans.BeanInstantiationException: Failed to instantiate [com.didiglobal.logi.job.core.Scheduler]: Factory method 'quartzScheduler' threw exception; nested exception is java.lang.NoClassDefFoundError: Could not initialize class com.didiglobal.logi.job.core.WorkerSingleton$Singleton
|
|
||||||
at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:657)
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
**问题原因：**
|
|
||||||
1. `KnowStreaming` 依赖的 `Logi-Job` 初始化 `WorkerSingleton$Singleton` 失败。
|
|
||||||
2. `WorkerSingleton$Singleton` 初始化的过程中，会去获取一些操作系统的信息，如果获取时出现了异常，则会导致 `WorkerSingleton$Singleton` 初始化失败。
|
|
||||||
|
|
||||||
|
|
||||||
**临时建议：**
|
|
||||||
|
|
||||||
`Logi-Job` 问题的修复时间不好控制，之前我们测试验证了一下，在 `Windows`、`Mac`、`CentOS` 这几个操作系统下基本上都是可以正常运行的。
|
|
||||||
|
|
||||||
所以，如果有条件的话，可以暂时先使用这几个系统部署 `KnowStreaming`。
|
|
||||||
|
|
||||||
如果在在 `Windows`、`Mac`、`CentOS` 这几个操作系统下也出现了启动失败的问题，可以重试2-3次看是否还是启动失败，或者换一台机器试试。
|
|
||||||
|
|
||||||
## 依赖ElasticSearch 8.0以上版本部署后指标信息无法正常显示如何解决
|
|
||||||
**错误现象**
|
|
||||||
```log
|
|
||||||
Warnings: [299 Elasticsearch-8.9.1-a813d015ef1826148d9d389bd1c0d781c6e349f0 "Legacy index templates are deprecated in favor of composable templates."]
|
|
||||||
```
|
|
||||||
**问题原因**
|
|
||||||
1. ES8.0和ES7.0版本存在Template模式的差异，建议使用 /_index_template 端点来管理模板；
|
|
||||||
2. ES java client在此版本的行为很奇怪表现为读取数据为空；
|
|
||||||
|
|
||||||
**解决方法**
|
|
||||||
修改`es_template_create.sh`脚本中所有的`/_template`为`/_index_template`后执行即可。
|
|
||||||
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
## 9.2、新旧版本对比
|
|
||||||
|
|
||||||
### 9.2.1、全新的设计理念
|
|
||||||
|
|
||||||
- 在 0 侵入、0 门槛的前提下提供直观 GUI 用于管理和观测 Apache Kafka®，帮助用户降低 Kafka CLI 操作门槛，轻松实现对原生 Kafka 集群的可管、可见、可掌控，提升 Kafka 使用体验和降低管理成本。
|
|
||||||
- 支持海量集群一键接入，无需任何改造，即可实现集群深度纳管，真正的 0 侵入、插件化系统设计，覆盖 0.10.x-3.x.x 众多 Kafka 版本无缝纳管。
|
|
||||||
|
|
||||||
### 9.2.2、产品名称&协议
|
|
||||||
|
|
||||||
- Know Streaming V3.0
|
|
||||||
|
|
||||||
- 名称：Know Streaming
|
|
||||||
- 协议：AGPL 3.0
|
|
||||||
|
|
||||||
- Logi-KM V2.x
|
|
||||||
|
|
||||||
- 名称：Logi-KM
|
|
||||||
- 协议：Apache License 2.0
|
|
||||||
|
|
||||||
### 9.2.3、功能架构
|
|
||||||
|
|
||||||
- Know Streaming V3.0
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
- Logi-KM V2.x
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 9.2.4、功能变更
|
|
||||||
|
|
||||||
- 多集群管理
|
|
||||||
|
|
||||||
- 增加健康监测体系、关键组件&指标 GUI 展示
|
|
||||||
- 增加 2.8.x 以上 Kafka 集群接入，覆盖 0.10.x-3.x
|
|
||||||
- 删除逻辑集群、共享集群、Region 概念
|
|
||||||
|
|
||||||
- Cluster 管理
|
|
||||||
|
|
||||||
- 增加集群概览信息、集群配置变更记录
|
|
||||||
- 增加 Cluster 健康分，健康检查规则支持自定义配置
|
|
||||||
- 增加 Cluster 关键指标统计和 GUI 展示，支持自定义配置
|
|
||||||
- 增加 Cluster 层 I/O、Disk 的 Load Reblance 功能，支持定时均衡任务（企业版）
|
|
||||||
- 删除限流、鉴权功能
|
|
||||||
- 删除 APPID 概念
|
|
||||||
|
|
||||||
- Broker 管理
|
|
||||||
|
|
||||||
- 增加 Broker 健康分
|
|
||||||
- 增加 Broker 关键指标统计和 GUI 展示，支持自定义配置
|
|
||||||
- 增加 Broker 参数配置功能，需重启生效
|
|
||||||
- 增加 Controller 变更记录
|
|
||||||
- 增加 Broker Datalogs 记录
|
|
||||||
- 删除 Leader Rebalance 功能
|
|
||||||
- 删除 Broker 优先副本选举
|
|
||||||
|
|
||||||
- Topic 管理
|
|
||||||
|
|
||||||
- 增加 Topic 健康分
|
|
||||||
- 增加 Topic 关键指标统计和 GUI 展示，支持自定义配置
|
|
||||||
- 增加 Topic 参数配置功能，可实时生效
|
|
||||||
- 增加 Topic 批量迁移、Topic 批量扩缩副本功能
|
|
||||||
- 增加查看系统 Topic 功能
|
|
||||||
- 优化 Partition 分布的 GUI 展示
|
|
||||||
- 优化 Topic Message 数据采样
|
|
||||||
- 删除 Topic 过期概念
|
|
||||||
- 删除 Topic 申请配额功能
|
|
||||||
|
|
||||||
- Consumer 管理
|
|
||||||
|
|
||||||
- 优化了 ConsumerGroup 展示形式，增加 Consumer Lag 的 GUI 展示
|
|
||||||
|
|
||||||
- ACL 管理
|
|
||||||
|
|
||||||
- 增加原生 ACL GUI 配置功能，可配置生产、消费、自定义多种组合权限
|
|
||||||
- 增加 KafkaUser 功能，可自定义新增 KafkaUser
|
|
||||||
|
|
||||||
- 消息测试（企业版）
|
|
||||||
|
|
||||||
- 增加生产者消息模拟器，支持 Data、Flow、Header、Options 自定义配置（企业版）
|
|
||||||
- 增加消费者消息模拟器，支持 Data、Flow、Header、Options 自定义配置（企业版）
|
|
||||||
|
|
||||||
- Job
|
|
||||||
|
|
||||||
- 优化 Job 模块，支持任务进度管理
|
|
||||||
|
|
||||||
- 系统管理
|
|
||||||
|
|
||||||
- 优化用户、角色管理体系，支持自定义角色配置页面及操作权限
|
|
||||||
- 优化审计日志信息
|
|
||||||
- 删除多租户体系
|
|
||||||
- 删除工单流程
|
|
||||||
@@ -1,848 +0,0 @@
|
|||||||
|
|
||||||
## 5.0、产品简介
|
|
||||||
|
|
||||||
`Know Streaming` 是一套云原生的 Kafka 管控平台，脱胎于众多互联网内部多年的 Kafka 运营实践经验，专注于 Kafka 运维管控、监控告警、资源治理、多活容灾等核心场景，在用户体验、监控、运维管控上进行了平台化、可视化、智能化的建设，提供一系列特色的功能，极大地方便了用户和运维人员的日常使用，让普通运维人员都能成为 Kafka 专家。
|
|
||||||
|
|
||||||
## 5.1、功能架构
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 5.2、体验路径
|
|
||||||
|
|
||||||
下面是用户第一次使用我们产品的典型体验路径：
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 5.3、常用功能
|
|
||||||
|
|
||||||
### 5.3.1、用户管理
|
|
||||||
|
|
||||||
用户管理是提供给管理员进行人员管理和用户角色管理的功能模块，可以进行新增用户和分配角色。下面是一个典型的场景：
|
|
||||||
eg：团队加入了新成员，需要给这位成员分配一个使用系统的账号，需要以下几个步骤
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“用户管理”>“人员管理”>“新增用户”，输入“账号”、“实名”、“密码”，根据此账号所需要的权限，选择此账号所对应的角色。如果有满足权限的角色，则用户新增成功。如果没有满足权限的角色，则需要新增角色（步骤 2）
|
|
||||||
- 步骤 2:点击“系统管理”>“用户管理”>“角色管理”>“新增角色”。输入角色名称和描述，给此角色分配权限，点击“确定”，角色新增成功
|
|
||||||
|
|
||||||
- 步骤 3:根据此新增的角色，参考步骤 1，重新新增用户
|
|
||||||
|
|
||||||
- 步骤 4:此用户账号新增成功，可以进行登录产品使用
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.2、接入集群
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“接入集群”
|
|
||||||
|
|
||||||
- 步骤 2:填写相关集群信息
|
|
||||||
|
|
||||||
- 集群名称：支持中英文、下划线、短划线（-），最长 128 字符。平台内不能重复
|
|
||||||
- Bootstrap Servers：输入 Bootstrap Servers 地址。输入完成之后会进行连接测试，测试完成之后会给出测试结果连接成功 or 连接失败（以及失败的原因）。
|
|
||||||
- Zookeeper：输入 zookeeper 地址，输入完成之后会进行连接测试，测试完成之后会给出测试结果连接成功 or 连接失败（以及失败的原因）
|
|
||||||
- Metrics 选填：JMX Port，输入 JMX 端口号；MaxConn，输入服务端最大允许的连接数
|
|
||||||
- Security：若有 JMX 账号密码，则输入账号密码
|
|
||||||
- Version：选择所支持的 kafka 版本，如果没有匹配则可以选择相近版本
|
|
||||||
- 集群配置选填：输入用户创建 kafka 客户端进行信息获取的相关配置
|
|
||||||
- 集群描述：最多 200 字符
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.3、新增 Topic
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“新增 Topic”按钮>“创建 Topic“抽屉
|
|
||||||
|
|
||||||
- 步骤 2:输入“Topic 名称（不能重复）”、“Topic 描述”、“分区数”、“副本数”、“数据保存时间”、“清理策略（删除或压缩）”
|
|
||||||
|
|
||||||
- 步骤 3:展开“更多配置”可以打开高级配置选项，根据自己需要输入相应配置参数
|
|
||||||
|
|
||||||
- 步骤 4:点击“确定”，创建 Topic 完成
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.4、Topic 扩分区
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“Topic 列表“>操作项”扩分区“>“扩分区”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:扩分区抽屉展示内容为“流量的趋势图”、“当前分区数及支持的最低消息写入速率”、“扩分区后支持的最低消息写入速率”
|
|
||||||
|
|
||||||
- 步骤 3:输入所需的分区总数，自动计算出扩分区后支持的最低消息写入速率
|
|
||||||
|
|
||||||
- 步骤 4:点击确定，扩分区完成
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.5、Topic 批量扩缩副本
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“批量操作下拉“>“批量扩缩副本“>“批量扩缩容”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:选择所需要进行扩缩容的 Topic，可多选，所选择的 Topic 出现在下方 Topic 列表中
|
|
||||||
|
|
||||||
- 步骤 3:Topic 列表展示 Topic“近三天平均流量”、“近三天峰值流量及时间”、“Partition 数”、”当前副本数“、“新副本数”
|
|
||||||
|
|
||||||
- 步骤 4:扩容时，选择目标节点，新增的副本会在选择的目标节点上；缩容时不需要选择目标节点，自动删除最后一个（或几个）副本
|
|
||||||
|
|
||||||
- 步骤 5:输入迁移任务配置参数，包含限流值和任务执行时间
|
|
||||||
|
|
||||||
- 步骤 6:输入任务描述
|
|
||||||
|
|
||||||
- 步骤 7:点击“确定”，创建 Topic 扩缩副本任务
|
|
||||||
|
|
||||||
- 步骤 8:去“Job”模块的 Job 列表查看创建的任务，如果已经执行则可以查看执行进度；如果未开始执行则可以编辑任务
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.6、Topic 批量迁移
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“批量操作下拉“>“批量迁移“>“批量迁移”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:选择所需要进行迁移的 Topic，可多选，所选择的 Topic 出现在下方 Topic 列表中
|
|
||||||
|
|
||||||
- 步骤 3:选择所需要迁移的 partition 和迁移数据的时间范围
|
|
||||||
|
|
||||||
- 步骤 4:选择目标节点（节点数必须不小于最大副本数）
|
|
||||||
|
|
||||||
- 步骤 5:点击“预览任务计划”，打开“任务计划”二次抽屉，可对目标 Broker ID 进行编辑
|
|
||||||
|
|
||||||
- 步骤 6:输入迁移任务配置参数，包含限流值和任务执行时间
|
|
||||||
|
|
||||||
- 步骤 7:输入任务描述
|
|
||||||
|
|
||||||
- 步骤 8:点击“确定”，创建 Topic 迁移任务
|
|
||||||
|
|
||||||
- 步骤 9:去“Job”模块的 Job 列表查看创建的任务，如果已经执行则可以查看执行进度；如果未开始执行则可以编辑任务
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.7、设置 Cluster 健康检查规则
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“集群健康状态旁边 icon”>“健康度设置抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:健康度设置抽屉展示出了检查项和其对应的权重，可以修改检查项的检查规则
|
|
||||||
|
|
||||||
- 步骤 3:检查规则可配置，分别为
|
|
||||||
|
|
||||||
- Cluster：集群 controller 数不等于 1（数字不可配置）不通过
|
|
||||||
- Broker：RequestQueueSize 大于等于 10（默认为 10，可配置数字）不通过
|
|
||||||
- Broker：NetworkProcessorAvgIdlePercent 的 Idle 小于等于 0.8%（默认为 0.8%，可配置数字）不通过
|
|
||||||
- Topic：无 leader 的 Topic 数量，大于等于 1（默认为 1，数字可配置）不通过
|
|
||||||
- Topic：Topic 在 10（默认为 10，数字可配置）个周期内 8（默认为 8，数字可配置）个周期内处于未同步的状态则不通过
|
|
||||||
- ConsumerGroup：Group 在 10（默认为 10，数字可配置）个周期内进行 8（默认为 8，数字可配置）次 re-balance 不通过
|
|
||||||
|
|
||||||
- 步骤 4:设置完成后，点击“确认”，健康检查规则设置成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.8、图表指标筛选
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“指标筛选 icon”>“指标筛选抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:指标筛选抽屉展示信息为以下几类“Health”、“Cluster”、“Broker”、“Consumer”、“Security”、“Job”
|
|
||||||
|
|
||||||
- 步骤 3:默认勾选比较重要的指标进行展示。根据需要选中/取消选中相应指标，点击”确认“，指标筛选成功，展示的图表随之变化
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.9、编辑 Broker 配置
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Brokers”>“Broker ID”>“Configuration”TAB>“编辑”按钮
|
|
||||||
|
|
||||||
- 步骤 2:输入配置项的新配置内容
|
|
||||||
|
|
||||||
- 步骤 3:（选填）点击“应用于全部 Broker”，将此配置项的修改应用于全部的 Broker
|
|
||||||
|
|
||||||
- 步骤 4:点击“确认”，Broker 配置修改成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.10、重置 consumer Offset
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Consumer”>“Consumer Group”名称>“Consumer Group 详情”抽屉>“重置 Offset”按钮>“重置 Offset”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:选择重置 Offset 的类型，可“重置到指定时间”或“重置分区”
|
|
||||||
|
|
||||||
- 步骤 3:重置到指定时间，可选择“最新 Offset”或“自定义时间”
|
|
||||||
|
|
||||||
- 步骤 4:重置分区，可选择 partition 和其重置的 offset
|
|
||||||
|
|
||||||
- 步骤 5:点击“确认”，重置 Offset 开始执行
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.3.11、新增 ACL
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Security”>“Users”>“新增 ACL”
|
|
||||||
|
|
||||||
- 步骤 2:输入 ACL 配置参数
|
|
||||||
|
|
||||||
- ACL 用途：生产权限、消费权限、自定义权限
|
|
||||||
- 生产权限时：可选择应用于所有 Kafka User 或者特定 Kafka User；可选择应用于所有 Topic 或者特定 Topic
|
|
||||||
- 消费权限时：可选择应用于所有 Kafka User 或者特定 Kafka User；可选择应用于所有 Topic 或者特定 Topic；可选择应用于所有 Consumer Group 或者特定 Consumer Group
|
|
||||||
|
|
||||||
- 步骤 3:点击“确定”，新增 ACL 成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 5.4、全部功能
|
|
||||||
|
|
||||||
### 5.4.1、登录/退出登录
|
|
||||||
|
|
||||||
- 登录：输入账号密码，点击登录
|
|
||||||
|
|
||||||
- 退出登录：鼠标悬停右上角“头像”或者“用户名”，出现小弹窗“登出”，点击“登出”，退出登录
|
|
||||||
|
|
||||||
### 5.4.2、系统管理
|
|
||||||
|
|
||||||
用户登录完成之后，点击页面右上角【系统管理】按钮，切换到系统管理的视角，可以进行配置管理、用户管理、审计日志查看。
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.1、配置管理
|
|
||||||
|
|
||||||
配置管理是提供给管理员一个快速配置配置文件的能力，所配置的配置文件将会在对应模块生效。
|
|
||||||
|
|
||||||
#### 5.4.2.2、查看配置列表
|
|
||||||
|
|
||||||
- 步骤 1:点击”系统管理“>“配置管理”
|
|
||||||
|
|
||||||
- 步骤 2:列表展示配置所属模块、配置键、配置值、启用状态、更新时间、更新人。列表有操作项编辑、删除，可对配置模块、配置键、配置值、描述、启用状态进行配置，也可删除此条配置
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.3、新增配置
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“配置管理”>“新增配置”
|
|
||||||
|
|
||||||
- 步骤 2：模块：下拉选择所有可配置的模块；配置键：不限制输入内容，500 字以内；配置值：代码编辑器样式，不限内容不限长度；启用状态开关：可以启用/禁用此项配置
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.4、编辑配置
|
|
||||||
|
|
||||||
可对配置模块、配置键、配置值、描述、启用状态进行配置。
|
|
||||||
|
|
||||||
#### 5.4.2.5、用户管理
|
|
||||||
|
|
||||||
用户管理是提供给管理员进行人员管理和用户角色管理的功能模块，可以进行新增用户和分配角色。
|
|
||||||
|
|
||||||
#### 5.4.2.6、人员管理列表
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“用户管理”>“人员管理”
|
|
||||||
|
|
||||||
- 步骤 2:人员管理列表展示用户角色、用户实名、用户分配的角色、更新时间、编辑操作。
|
|
||||||
|
|
||||||
- 步骤 3:列表支持”用户账号“、“用户实名”、“角色名”筛选。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.7、新增用户
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“用户管理”>“人员管理”>“新增用户”
|
|
||||||
|
|
||||||
- 步骤 2:填写“用户账号”、“用户实名”、“用户密码”这些必填参数，可以对此账号分配已经存在的角色。
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.8、编辑用户
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“用户管理”>“人员管理”>列表操作项“编辑”
|
|
||||||
|
|
||||||
- 步骤 2:用户账号不可编辑；可以编辑“用户实名”，修改“用户密码”，重新分配“用户角色“
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.9、角色管理列表
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“用户管理”>“角色管理”
|
|
||||||
|
|
||||||
- 步骤 2:角色列表展示信息为“角色 ID”、“名称”、“描述”、“分配用户数”、“最后修改人”、“最后更新时间”、操作项“查看详情”、操作项”分配用户“
|
|
||||||
|
|
||||||
- 步骤 3:列表有筛选框，可对“角色名称”进行筛选
|
|
||||||
|
|
||||||
- 步骤 4:列表操作项，“查看详情”可查看到角色绑定的权限项，”分配用户“可对此项角色下绑定的用户进行增减
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.10、新增角色
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“用户管理”>“角色管理”>“新增角色”
|
|
||||||
|
|
||||||
- 步骤 2:输入“角色名称”（角色名称只能由中英文大小写、数字、下划线\_组成，长度限制在 3 ～ 128 字符）、“角色描述“（不能为空）、“分配权限“（至少需要分配一项权限），点击确认，新增角色成功添加到角色列表
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.2.11、审计日志
|
|
||||||
|
|
||||||
- 步骤 1:点击“系统管理”>“审计日志“
|
|
||||||
- 步骤 2:审计日志包含所有对于系统的操作记录，操作记录列表展示信息为下
|
|
||||||
|
|
||||||
- “模块”：操作对象所属的功能模块
|
|
||||||
- “操作对象”：具体哪一个集群、任务 ID、topic、broker、角色等
|
|
||||||
- “行为”：操作记录的行为，包含“新增”、“替换”、“读取”、“禁用”、“修改”、“删除”、“编辑”等
|
|
||||||
- “操作内容”：具体操作的内容是什么
|
|
||||||
- “操作时间”：操作发生的时间
|
|
||||||
- “操作人”：此项操作所属的用户
|
|
||||||
|
|
||||||
- 步骤 3:操作记录列表可以对“模块“、”操作对象“、“操作内容”、”操作时间“进行筛选
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.3、多集群管理
|
|
||||||
|
|
||||||
#### 5.4.3.1、多集群列表
|
|
||||||
|
|
||||||
- 步骤 1:点击顶部导航栏“多集群管理”
|
|
||||||
|
|
||||||
- 步骤 2:多集群管理页面包含的信息为：”集群信息总览“、“集群列表”、“列表筛选项”、“接入集群”
|
|
||||||
|
|
||||||
- 步骤 3:集群列表筛选项为
|
|
||||||
|
|
||||||
- 集群信息总览：cluster 总数、live 数、down 数
|
|
||||||
- 版本筛选：包含所有存在的集群版本
|
|
||||||
- 健康分筛选：筛选项为 0、10、20、30、40、50、60、70、80、90、100
|
|
||||||
- live、down 筛选：多选
|
|
||||||
- 下拉框筛选排序，选项维度为“接入时间”、“健康分“、”Messages“、”MessageSize“、”BytesIn“、”BytesOut“、”Brokers“；可对这些维度进行“升序”、“降序”排序
|
|
||||||
|
|
||||||
- 步骤 4:每个卡片代表一个集群，其所展示的集群概览信息包括“健康分及健康检查项通过数”、“broker 数量”、“ZK 数量”、“版本号”、“BytesIn 均衡状态”、“BytesOut 均衡状态”、“Disk 均衡状态”、”Messages“、“MessageSize”、“BytesIn”、“BytesOut”、“接入时间”
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.3.2、接入集群
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“接入集群”
|
|
||||||
|
|
||||||
- 步骤 2:填写相关集群信息
|
|
||||||
- 集群名称：平台内不能重复
|
|
||||||
- Bootstrap Servers：输入 Bootstrap Servers 地址，输入完成之后会进行连接测试，测试完成之后会给出测试结果连接成功 or 连接失败（以及失败的原因）。
|
|
||||||
- Zookeeper：输入 zookeeper 地址，输入完成之后会进行连接测试，测试完成之后会给出测试结果连接成功 or 连接失败（以及失败的原因）
|
|
||||||
- Metrics 选填：JMX Port，输入 JMX 端口号；MaxConn，输入服务端最大允许的连接数
|
|
||||||
- Security：若有 JMX 账号密码，则输入账号密码
|
|
||||||
- Version：kafka 版本，如果没有匹配则可以选择相近版本
|
|
||||||
- 集群配置选填：用户创建 kafka 客户端进行信息获取的相关配置
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.3.3、删除集群
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>鼠标悬浮集群卡片>点击卡片右上角“删除 icon”>打开“删除弹窗”
|
|
||||||
|
|
||||||
- 步骤 2:在删除弹窗中的“集群名称”输入框，输入所要删除集群的集群名称，点击“删除”，成功删除集群，解除平台的纳管关系（集群资源不会删除）
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.4、Cluster 管理
|
|
||||||
|
|
||||||
#### 5.4.4.1、Cluster Overview
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>进入单集群管理界面
|
|
||||||
|
|
||||||
- 步骤 2:左侧导航栏
|
|
||||||
|
|
||||||
- 一级导航：Cluster；二级导航：Overview、Load Rebalance
|
|
||||||
- 一级导航：Broker；二级导航：Overview、Brokers、Controller
|
|
||||||
- 一级导航：Topic；二级导航：Overview、Topics
|
|
||||||
- 一级导航：Consumer
|
|
||||||
- 一级导航：Testing；二级导航：Produce、Consume
|
|
||||||
- 一级导航：Security；二级导航：ACLs、Users
|
|
||||||
- 一级导航：Job
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.2、查看 Cluster 概览信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”
|
|
||||||
|
|
||||||
- 步骤 2:cluster 概览信息包括以下内容
|
|
||||||
|
|
||||||
- 集群健康分，健康检查通过项
|
|
||||||
- Cluster 信息：包含名称、版本、均衡状态
|
|
||||||
- Broker 信息：Broker 总数、controller 信息、similar config 信息
|
|
||||||
- Topic 信息：Topic 总数、No Leader、<Min ISR、URP
|
|
||||||
- Consumer Group 信息：Consumer Group 总数、是否存在 Dead 情况
|
|
||||||
- 指标图表
|
|
||||||
- 历史变更记录：名称、时间、内容、类型
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.3、设置 Cluster 健康检查规则
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“集群健康状态旁边 icon”>“健康度设置抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:健康度设置抽屉展示出了检查项和其对应的权重，可以修改检查项的检查规则
|
|
||||||
|
|
||||||
- 步骤 3:检查规则可配置，分别为
|
|
||||||
|
|
||||||
- Cluster：集群 controller 数不等于 1（数字不可配置）不通过
|
|
||||||
- Broker：RequestQueueSize 大于等于 10（默认为 10，可配置数字）不通过
|
|
||||||
- Broker：NetworkProcessorAvgIdlePercent 的 Idle 小于等于 0.8%（默认为 0.8%，可配置数字）不通过
|
|
||||||
- Topic：无 leader 的 Topic 数量，大于等于 1（默认为 1，数字可配置）不通过
|
|
||||||
- Topic：Topic 在 10（默认为 10，数字可配置）个周期内 8（默认为 8，数字可配置）个周期内处于未同步的状态
|
|
||||||
- ConsumerGroup：Group 在 10（默认为 10，数字可配置）个周期内进行 8（默认为 8，数字可配置）次 re-balance 不通过
|
|
||||||
|
|
||||||
- 步骤 4:设置完成后，点击“确认”，健康检查规则设置成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.4、查看 Cluster 健康检查详情
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“集群健康状态旁边【查看详情】”>“健康检查详情抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:健康检查详情抽屉展示信息为：“检查模块”、“检查项”、“权重”、“得分”、“检查时间”、“检查结果是否通过”，若未通过会展示未通过的对象
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.5、编辑 Cluster 信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“Cluster 名称旁边编辑 icon”>“编辑集群抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:可编辑的信息包括“集群名称”、“Bootstrap Servers”、“Zookeeper”、“JMX Port”、“Maxconn（最大连接数）”、“Security（认证措施）”、“Version（版本号）”、“集群配置”、“集群描述”
|
|
||||||
|
|
||||||
- 步骤 3:点击“确定”，成功编辑集群信息
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.6、图表指标筛选
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“指标筛选 icon”>“指标筛选抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:指标筛选抽屉展示信息为以下几类“Health”、“Cluster”、“Broker”、“Consumer”、“Security”、“Job”
|
|
||||||
|
|
||||||
- 步骤 3:默认勾选比较重要的指标进行展示。根据需要选中/取消选中相应指标，点击”确认“，指标筛选成功，展示的图表随之变化
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.7、图表时间筛选
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“时间选择下拉框”>“时间选择弹窗”
|
|
||||||
|
|
||||||
- 步骤 2:选择时间“最近 15 分钟”、“最近 1 小时”、“最近 6 小时”、“最近 12 小时”、“最近 1 天”，也可以自定义时间段范围
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.4.8、查看集群历史变更记录
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Overview”>“历史变更记录”区域
|
|
||||||
|
|
||||||
- 步骤 2:历史变更记录区域展示了历史的配置变更，每条记录可展开收起。包含“配置对象”、“变更时间”、“变更内容”、“配置类型”
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.5、Load Rebalance（企业版）
|
|
||||||
|
|
||||||
#### 5.4.5.1、查看 Load Rebalance 概览信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Load Rebalance”
|
|
||||||
|
|
||||||
- 步骤 2:Load Rebalance 概览信息包含“均衡状态卡片”、“Disk 信息卡片”、“BytesIn 信息卡片”、“BytesOut 信息卡片”、“Broker 均衡状态列表”
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.5.2、设置集群规格
|
|
||||||
|
|
||||||
提供对集群的每个节点的 Disk、BytesIn、BytesOut 的规格进行设置的功能
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Load Rebalance”>“State 卡片 icon“>”设置集群规格抽屉“
|
|
||||||
|
|
||||||
- 步骤 2:穿梭框左侧展示集群中的待选节点，穿梭框右侧展示已经选中的节点，选择自己所需设置规格的节点
|
|
||||||
|
|
||||||
- 步骤 3:设置“单机核数”、“单机磁盘”、“单机网络”，点击确定，完成设置
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.5.3、均衡状态列表筛选
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Load Rebalance”>“筛选列表”按钮>筛选弹窗
|
|
||||||
|
|
||||||
- 步骤 2:可选择“Disk”、“BytesIn”、“BytesOut”三种维度，其各自对应“已均衡”、“未均衡”两种状态，可以组合进行筛选
|
|
||||||
|
|
||||||
- 步骤 3:点击“确认”，执行筛选操作
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.5.4、立即均衡
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Load Rebalance”>“立即均衡”按钮>“立即均衡抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:配置均衡策略
|
|
||||||
|
|
||||||
- 指标计算周期：默认近 10mins，可选择
|
|
||||||
- 均衡维度：默认 Disk、BytesIn、BytesOut，可选择
|
|
||||||
- 均衡区间：在表格内自定义配置均衡区间范围（单位：%，大于 0，小于 100）
|
|
||||||
- Topic 黑名单：选择 topic 黑名单。通过穿梭框（支持模糊选择）选出目标 topic（本次均衡，略过已选的 topic）
|
|
||||||
|
|
||||||
- 步骤 3:配置运行参数
|
|
||||||
|
|
||||||
- 吞吐量优先：并行度 0（无限制）， 策略是优先执行大小最大副本
|
|
||||||
- 稳定性优先： 并行度 1 ，策略是优先执行大小最小副本
|
|
||||||
- 自定义：可以自由设置并行度和优先执行的副本策略
|
|
||||||
- 限流值：流量最大值，0-99999 自定义
|
|
||||||
|
|
||||||
- 步骤 4:点击“预览计划”按钮，打开执行计划弹窗。可以看到计划概览信息、计划明细信息
|
|
||||||
|
|
||||||
- 步骤 5:点击“预览计划弹窗”的“执行文件”，可以下载 json 格式的执行文件
|
|
||||||
|
|
||||||
- 步骤 6:点击“预览计划弹窗”的“立即均衡”按钮，开始执行均衡任务
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.5.5、周期均衡
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Cluster”>“Load Rebalance”>“周期均衡”按钮>“周期均衡抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:配置均衡策略
|
|
||||||
|
|
||||||
- 指标计算周期：默认近 10mins，可选择
|
|
||||||
- 均衡维度：默认 Disk、BytesIn、BytesOut，可选择
|
|
||||||
- 均衡区间：在表格内自定义配置均衡区间范围（单位：%，大于 0，小于 100）
|
|
||||||
- Topic 黑名单：选择 topic 黑名单。通过穿梭框（支持模糊选择）选出目标 topic（本次均衡，略过已选的 topic）
|
|
||||||
|
|
||||||
- 步骤 3:配置运行参数
|
|
||||||
|
|
||||||
- 任务并行度：每个节点同时迁移的副本数量
|
|
||||||
- 任务周期：时间选择器，自定义选择运行周期
|
|
||||||
- 稳定性优先： 并行度 1 ，策略是优先执行大小最小副本
|
|
||||||
- 自定义：可以自由设置并行度和优先执行的副本策略
|
|
||||||
- 限流值：流量最大值，0-99999 自定义
|
|
||||||
|
|
||||||
- 步骤 4:点击“预览计划”按钮，打开执行计划弹窗。可以看到计划概览信息、计划明细信息
|
|
||||||
|
|
||||||
- 步骤 5:点击“预览计划弹窗”的“执行文件”，可以下载 json 格式的执行文件
|
|
||||||
|
|
||||||
- 步骤 6:点击“预览计划弹窗”的“立即均衡”按钮，开始执行均衡任务
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.6、Broker
|
|
||||||
|
|
||||||
#### 5.4.6.1、查看 Broker 概览信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Broker”>“Overview”
|
|
||||||
|
|
||||||
- 步骤 2:Broker 概览信息包括以下内容
|
|
||||||
|
|
||||||
- 集群健康分，健康检查通过项
|
|
||||||
- Broker 信息：包含名称、版本、均衡状态
|
|
||||||
- Broker 信息：Broker 总数、controller 信息、similar config 信息
|
|
||||||
- Topic 信息：Topic 总数、No Leader、<Min ISR、URP
|
|
||||||
- Consumer Group 信息：Consumer Group 总数、是否存在 Dead 情况
|
|
||||||
- 指标图表
|
|
||||||
- 历史变更记录：名称、时间、内容、类型
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.6.2、编辑 Broker 配置
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Brokers”>“Broker ID”>“Configuration”TAB>“编辑”按钮
|
|
||||||
|
|
||||||
- 步骤 2:输入配置项的新配置内容
|
|
||||||
|
|
||||||
- 步骤 3:（选填）点击“应用于全部 Broker”，将此配置项的修改应用于全部的 Broker
|
|
||||||
|
|
||||||
- 步骤 4:点击“确认”，Broker 配置修改成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.6.3、查看 Broker DataLogs
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Brokers”>“Broker ID”>“Data Logs”TAB>“编辑”按钮
|
|
||||||
|
|
||||||
- 步骤 2:Broker DataLogs 列表展示的信息为“Folder”、“topic”、“Partition”、“Offset Lag”、“Size”
|
|
||||||
|
|
||||||
- 步骤 3:输入框输入”Topic Name“可以筛选结果
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.6.4、查看 Controller 列表
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Broker”>“Controller”
|
|
||||||
|
|
||||||
- 步骤 2:Controller 列表展示的信息为“Change Time”、“Broker ID”、“Broker Host”
|
|
||||||
|
|
||||||
- 步骤 3:输入框输入“Broker Host“可以筛选结果
|
|
||||||
|
|
||||||
- 步骤 4:点击 Broker ID 可以打开 Broker 详情，进行修改配置或者查看 DataLogs
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.7、Topic
|
|
||||||
|
|
||||||
#### 5.4.7.1、查看 Topic 概览信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Overview”
|
|
||||||
|
|
||||||
- 步骤 2:Topic 概览信息包括以下内容
|
|
||||||
|
|
||||||
- 集群健康分，健康检查通过项
|
|
||||||
- Topics：Topic 总数
|
|
||||||
- Partitions：Partition 总数
|
|
||||||
- PartitionNoLeader：没有 leader 的 partition 个数
|
|
||||||
- < Min ISR：同步副本数小于 Min ISR
|
|
||||||
- =Min ISR：同步副本数等于 Min ISR
|
|
||||||
- Topic 指标图表
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.2、查看 Topic 健康检查详情
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Overview”>“集群健康状态旁边【查看详情】”>“健康检查详情抽屉”
|
|
||||||
|
|
||||||
- 步骤 2:健康检查详情抽屉展示信息为：“检查项”、“权重”、“得分”、“检查时间”、“检查结果是否通过”，若未通过会展示未通过的对象
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.3、查看 Topic 列表
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”
|
|
||||||
|
|
||||||
- 步骤 2:Topic 列表展示内容为“TopicName”、“Partitions”、“Replications”、“健康分”、“BytesIn”、“BytesOut”、“MessageSize”、“保存时间”、“描述”、操作项”扩分区“、操作项”删除“
|
|
||||||
|
|
||||||
- 步骤 3:筛选框输入“TopicName”可以对列表进行筛选；点击“展示系统 Topic”开关，可以筛选系统 topic
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.4、新增 Topic
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“新增 Topic”按钮>“创建 Topic“抽屉
|
|
||||||
|
|
||||||
- 步骤 2:输入“Topic 名称（不能重复）”、“Topic 描述”、“分区数”、“副本数”、“数据保存时间”、“清理策略（删除或压缩）”
|
|
||||||
|
|
||||||
- 步骤 3:展开“更多配置”可以打开高级配置选项，根据自己需要输入相应配置参数
|
|
||||||
|
|
||||||
- 步骤 4:点击“确定”，创建 Topic 完成
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.5、Topic 扩分区
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“Topic 列表“>操作项”扩分区“>“扩分区”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:扩分区抽屉展示内容为“流量的趋势图”、“当前分区数及支持的最低消息写入速率”、“扩分区后支持的最低消息写入速率”
|
|
||||||
|
|
||||||
- 步骤 3:输入所需的分区总数，自动计算出扩分区后支持的最低消息写入速率
|
|
||||||
|
|
||||||
- 步骤 4:点击确定，扩分区完成
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.6、删除 Topic
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“Topic 列表“>操作项”删除“>“删除 Topic”弹窗
|
|
||||||
|
|
||||||
- 步骤 2:输入“TopicName”进行二次确认
|
|
||||||
|
|
||||||
- 步骤 3:点击“删除”，删除 Topic 完成
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.7、Topic 批量扩缩副本
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“批量操作下拉“>“批量扩缩副本“>“批量扩缩容”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:选择所需要进行扩缩容的 Topic，可多选，所选择的 Topic 出现在下方 Topic 列表中
|
|
||||||
|
|
||||||
- 步骤 3:Topic 列表展示 Topic“近三天平均流量”、“近三天峰值流量及时间”、“Partition 数”、”当前副本数“、“新副本数”
|
|
||||||
|
|
||||||
- 步骤 4:扩容时，选择目标节点，新增的副本会在选择的目标节点上；缩容时不需要选择目标节点，自动删除最后一个（或几个）副本
|
|
||||||
|
|
||||||
- 步骤 5:输入迁移任务配置参数，包含限流值和任务执行时间
|
|
||||||
|
|
||||||
- 步骤 6:输入任务描述
|
|
||||||
|
|
||||||
- 步骤 7:点击“确定”，执行 Topic 扩缩容任务
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.7.8、Topic 批量迁移
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Topic”>“Topics”>“批量操作下拉“>“批量迁移“>“批量迁移”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:选择所需要进行迁移的 Topic，可多选，所选择的 Topic 出现在下方 Topic 列表中
|
|
||||||
|
|
||||||
- 步骤 3:选择所需要迁移的 partition 和迁移数据的时间范围
|
|
||||||
|
|
||||||
- 步骤 4:选择目标节点（节点数必须不小于最大副本数）
|
|
||||||
|
|
||||||
- 步骤 5:点击“预览任务计划”，打开“任务计划”二次抽屉，可对每个 partition 的目标 Broker ID 进行编辑，目标 broker 应该等于副本数
|
|
||||||
|
|
||||||
- 步骤 6:输入迁移任务配置参数，包含限流值和任务执行时间
|
|
||||||
|
|
||||||
- 步骤 7:输入任务描述
|
|
||||||
|
|
||||||
- 步骤 8:点击“确定”，执行 Topic 迁移任务
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.8、Consumer
|
|
||||||
|
|
||||||
#### 5.4.8.1、Consumer Overview
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Consumer”
|
|
||||||
|
|
||||||
- 步骤 2:Consumer 概览信息包括以下内容
|
|
||||||
|
|
||||||
- 集群健康分，健康检查通过项
|
|
||||||
- Groups：Consumer Group 总数
|
|
||||||
- GroupsActives：活跃的 Group 总数
|
|
||||||
- GroupsEmptys：Empty 的 Group 总数
|
|
||||||
- GroupRebalance：进行 Rebalance 的 Group 总数
|
|
||||||
- GroupDeads：Dead 的 Group 总数
|
|
||||||
- Consumer Group 列表
|
|
||||||
|
|
||||||
- 步骤 3:输入“Consumer Group”、“Topic Name‘，可对列表进行筛选
|
|
||||||
|
|
||||||
- 步骤 4:点击列表“Consumer Group”名称，可以查看 Comsuer Group 详情
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.8.2、查看 Consumer 列表
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Consumer”>“Consumer Group”名称>“Consumer Group 详情”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:Consumer Group 详情有列表视图和图表视图
|
|
||||||
|
|
||||||
- 步骤 3:列表视图展示信息为 Consumer 列表，包含”Topic Partition“、”Member ID“、”Current Offset“、“Log End Offset”、”Lag“、”Host“、”Client ID“
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.8.3、重置 Offset
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Consumer”>“Consumer Group”名称>“Consumer Group 详情”抽屉>“重置 Offset”按钮>“重置 Offset”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:选择重置 Offset 的类型，可“重置到指定时间”或“重置分区”
|
|
||||||
|
|
||||||
- 步骤 3:重置到指定时间，可选择“最新 Offset”或“自定义时间”
|
|
||||||
|
|
||||||
- 步骤 4:重置分区，可选择 partition 和其重置的 offset
|
|
||||||
|
|
||||||
- 步骤 5:点击“确认”，重置 Offset 开始执行
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.9、Testing（企业版）
|
|
||||||
|
|
||||||
#### 5.4.9.1、生产测试
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Testing”>“Produce”
|
|
||||||
|
|
||||||
- 步骤 2:生产配置
|
|
||||||
|
|
||||||
- Data：选择数据写入的 topic，输入写入数据的 key（暂只支持 string 格式），输入写入数据的 value（暂只支持 string 格式）。其中 key 和 value 可以随机生成
|
|
||||||
- Flow：输入单次发送的消息数量，默认为 1，可以手动修改。选择手动生产模式，代表每次点击按钮【Run】执行生产；选择周期生产模式，需要填写运行总时间和运行时间间隔。
|
|
||||||
- Header：输入 Header 的 key，value
|
|
||||||
- Options：选择 Froce Partition，代表消息仅发送到这些选择的 Partition。选择数据压缩格式。选择 Acks 参数，none 意思是消息发送了就认为发送成功；leader 意思是 leader 接收到消息（不管 follower 有没有同步成功）认为消息发送成功；all 意思是所有的 follower 消息同步成功认为是消息发送成功
|
|
||||||
|
|
||||||
- 步骤 3:点击按钮【Run】，生产测试开始，可以从右侧看到生产测试的信息
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.9.2、消费测试
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Testing”>“Consume”
|
|
||||||
|
|
||||||
- 步骤 2:消费配置
|
|
||||||
|
|
||||||
- Topic：选择数据从哪个 topic 进行消费
|
|
||||||
- Start From：选择数据从什么地方开始消费，可以根据时间选择或者根据 Offset 进行选择
|
|
||||||
- Until：选择消费截止到什么地方，可以根据时间或者 offset 或者消息数等进行选择
|
|
||||||
- Filter：选择过滤器的规则。包含/不包含某【key，value】；等于/大于/小于多少条消息
|
|
||||||
|
|
||||||
- 步骤 3:点击按钮【Run】，消费测试开始，可以在右边看到消费的明细信息
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.10、Security
|
|
||||||
|
|
||||||
注意：只有在开启集群认证的情况下才能够使用 Security 功能
|
|
||||||
|
|
||||||
#### 5.4.10.1、查看 ACL 概览信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Security”>“ACLs”
|
|
||||||
|
|
||||||
- 步骤 2:ACL 概览信息包括以下内容
|
|
||||||
|
|
||||||
- Enable：是否可用
|
|
||||||
- ACLs：ACL 总数
|
|
||||||
- Users：User 总数
|
|
||||||
- Topics：Topic 总数
|
|
||||||
- Consumer Groups：Consumer Group 总数
|
|
||||||
- ACL 列表
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.10.2、新增 ACl
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Security”>“Users”>“新增 ACL”
|
|
||||||
|
|
||||||
- 步骤 2:输入 ACL 配置参数
|
|
||||||
|
|
||||||
- ACL 用途：生产权限、消费权限、自定义权限
|
|
||||||
- 生产权限时：可选择应用于所有 Kafka User 或者特定 Kafka User；可选择应用于所有 Topic 或者特定 Topic
|
|
||||||
- 消费权限时：可选择应用于所有 Kafka User 或者特定 Kafka User；可选择应用于所有 Topic 或者特定 Topic；可选择应用于所有 Consumer Group 或者特定 Consumer Group
|
|
||||||
|
|
||||||
- 步骤 3:点击“确定”，新增 ACL 成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.10.3、查看 User 信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Security”>“ACLs”
|
|
||||||
|
|
||||||
- 步骤 2:User 列表展示内容包括“Kafka User 名称”、“认证方式”、“passwprd”、操作项”修改密码“、”操作项“删除”
|
|
||||||
|
|
||||||
- 步骤 3:筛选框输入“Kafka User”可筛选出列表中相关 Kafka User
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.10.4、新增 Kafka User
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Security”>“Users”>“新增 Kafka User”
|
|
||||||
|
|
||||||
- 步骤 2:输入 Kafka User 名称、认证方式、密码
|
|
||||||
|
|
||||||
- 步骤 3:点击“确定”，新增 Kafka User 成功
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 5.4.11、Job
|
|
||||||
|
|
||||||
#### 5.4.11.1、查看 Job 概览信息
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Job“
|
|
||||||
|
|
||||||
- 步骤 2:Job 概览信息包括以下内容
|
|
||||||
|
|
||||||
- Jobs：Job 总数
|
|
||||||
- Doing：正在运行的 Job 总数
|
|
||||||
- Prepare：准备运行的 Job 总数
|
|
||||||
- Success：运行成功的 Job 总数
|
|
||||||
- Fail：运行失败的 Job 总数
|
|
||||||
- Job 列表
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.11.2、Job 查看进度
|
|
||||||
|
|
||||||
Doing 状态下的任务可以查看进度
|
|
||||||
|
|
||||||
- 步骤 1:点击“多集群管理”>“集群卡片”>“Job”>“Job”列表>操作项“查看进度”>“查看进度”抽屉
|
|
||||||
|
|
||||||
- 步骤 2:
|
|
||||||
|
|
||||||
- 均衡任务：任务基本信息、均衡计划、任务执行明细信息
|
|
||||||
- 扩缩副本：任务基本信息、任务执行明细信息、节点流量情况
|
|
||||||
- Topic 迁移：任务基本信息、任务执行明细信息、节点流量情况
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### 5.4.11.3、Job 编辑任务
|
|
||||||
|
|
||||||
Prepare 状态下的任务可以进行编辑
|
|
||||||
|
|
||||||
- 点击“多集群管理”>“集群卡片”>“Job”>“Job”列表>操作项“编辑”
|
|
||||||
|
|
||||||
- 对任务执行的参数进行重新配置
|
|
||||||
|
|
||||||
- 集群均衡：可以对指标计算周期、均衡维度、topic 黑名单、运行配置等参数重新设置
|
|
||||||
- Topic 迁移：可以对 topic 需要迁移的 partition、迁移数据的时间范围、目标 broker 节点、限流值、执行时间、描述等参数重新配置
|
|
||||||
- topic 扩缩副本：可以对最终副本数、限流值、任务执行时间、描述等参数重新配置
|
|
||||||
|
|
||||||
- 点击“确定”，编辑任务成功
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<artifactId>km-biz</artifactId>
|
|
||||||
<version>${revision}</version>
|
|
||||||
<packaging>jar</packaging>
|
|
||||||
|
|
||||||
<parent>
|
|
||||||
<artifactId>km</artifactId>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<version>${revision}</version>
|
|
||||||
</parent>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<!-- maven properties -->
|
|
||||||
<maven.test.skip>true</maven.test.skip>
|
|
||||||
<downloadSources>true</downloadSources>
|
|
||||||
|
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
||||||
<file_encoding>UTF-8</file_encoding>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<artifactId>km-core</artifactId>
|
|
||||||
<version>${project.parent.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<artifactId>km-rebalance</artifactId>
|
|
||||||
<version>${project.parent.version}</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- spring -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework</groupId>
|
|
||||||
<artifactId>spring-web</artifactId>
|
|
||||||
<version>${spring.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework</groupId>
|
|
||||||
<artifactId>spring-test</artifactId>
|
|
||||||
<version>${spring.version}</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- javax -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>javax.servlet</groupId>
|
|
||||||
<artifactId>javax.servlet-api</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>javax.annotation</groupId>
|
|
||||||
<artifactId>javax.annotation-api</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.kafka</groupId>
|
|
||||||
<artifactId>kafka-clients</artifactId>
|
|
||||||
<version>${kafka-clients.version}</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>commons-lang</groupId>
|
|
||||||
<artifactId>commons-lang</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>commons-codec</groupId>
|
|
||||||
<artifactId>commons-codec</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.eclipse.jetty</groupId>
|
|
||||||
<artifactId>jetty-util</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
|
||||||
<artifactId>caffeine</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<!-- json -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.alibaba</groupId>
|
|
||||||
<artifactId>fastjson</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.fasterxml.jackson.core</groupId>
|
|
||||||
<artifactId>jackson-databind</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework</groupId>
|
|
||||||
<artifactId>spring-context</artifactId>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
|
||||||
</project>
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.broker;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaBrokerConfigModifyParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.config.kafka.KafkaBrokerConfigVO;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public interface BrokerConfigManager {
|
|
||||||
/**
|
|
||||||
* 获取Broker配置详细信息
|
|
||||||
* @param clusterPhyId 物理集群ID
|
|
||||||
* @param brokerId brokerId
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<List<KafkaBrokerConfigVO>> getBrokerConfigDetail(Long clusterPhyId, Integer brokerId);
|
|
||||||
|
|
||||||
Result<Void> modifyBrokerConfig(KafkaBrokerConfigModifyParam modifyParam, String operator);
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.broker;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.broker.BrokerBasicVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.log.LogDirVO;
|
|
||||||
|
|
||||||
public interface BrokerManager {
|
|
||||||
Result<BrokerBasicVO> getBrokerBasic(Long clusterPhyId, Integer brokerId);
|
|
||||||
|
|
||||||
PaginationResult<LogDirVO> getBrokerLogDirs(Long clusterPhyId, Integer brokerId, PaginationBaseDTO dto);
|
|
||||||
}
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.broker.impl;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.broker.BrokerConfigManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaConfigDetail;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaBrokerConfigModifyParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerConfigPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.config.kafka.KafkaBrokerConfigVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.config.ConfigDiffTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import org.apache.kafka.common.config.ConfigDef;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class BrokerConfigManagerImpl implements BrokerConfigManager {
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerConfigService brokerConfigService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<KafkaBrokerConfigVO>> getBrokerConfigDetail(Long clusterPhyId, Integer brokerId) {
|
|
||||||
// 获取当前broker配置
|
|
||||||
Result<List<KafkaConfigDetail>> configResult = brokerConfigService.getBrokerConfigDetailFromKafka(clusterPhyId, brokerId);
|
|
||||||
if (configResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(configResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 获取差异的配置
|
|
||||||
List<BrokerConfigPO> diffPOList = brokerConfigService.getBrokerConfigDiffFromDB(clusterPhyId, brokerId);
|
|
||||||
|
|
||||||
// 组装数据
|
|
||||||
return Result.buildSuc(this.convert2KafkaBrokerConfigVOList(configResult.getData(), diffPOList));
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<KafkaBrokerConfigVO> convert2KafkaBrokerConfigVOList(List<KafkaConfigDetail> configList, List<BrokerConfigPO> diffPOList) {
|
|
||||||
if (ValidateUtils.isEmptyList(configList)) {
|
|
||||||
return new ArrayList<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<String, BrokerConfigPO> poMap = diffPOList.stream().collect(Collectors.toMap(BrokerConfigPO::getConfigName, Function.identity()));
|
|
||||||
|
|
||||||
List<KafkaBrokerConfigVO> voList = ConvertUtil.list2List(configList, KafkaBrokerConfigVO.class);
|
|
||||||
for (KafkaBrokerConfigVO vo: voList) {
|
|
||||||
BrokerConfigPO po = poMap.get(vo.getName());
|
|
||||||
if (po != null) {
|
|
||||||
vo.setExclusive(po.getDiffType().equals(ConfigDiffTypeEnum.ALONE_POSSESS.getCode()));
|
|
||||||
vo.setDifferentiated(po.getDiffType().equals(ConfigDiffTypeEnum.UN_EQUAL.getCode()));
|
|
||||||
} else {
|
|
||||||
vo.setExclusive(false);
|
|
||||||
vo.setDifferentiated(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
ConfigDef.ConfigKey configKey = KafkaConstant.KAFKA_ALL_CONFIG_DEF_MAP.get(vo.getName());
|
|
||||||
if (configKey == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
vo.setDocumentation(configKey.documentation);
|
|
||||||
vo.setDefaultValue(configKey.defaultValue.toString());
|
|
||||||
} catch (Exception e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return voList;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> modifyBrokerConfig(KafkaBrokerConfigModifyParam modifyParam, String operator) {
|
|
||||||
if (modifyParam.getApplyAll() == null || !modifyParam.getApplyAll()) {
|
|
||||||
return brokerConfigService.modifyBrokerConfig(modifyParam, operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<Broker> brokerList = brokerService.listAliveBrokersFromDB(modifyParam.getClusterPhyId());
|
|
||||||
for (Broker broker: brokerList) {
|
|
||||||
modifyParam.setBrokerId(broker.getBrokerId());
|
|
||||||
Result<Void> rv = brokerConfigService.modifyBrokerConfig(modifyParam, operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.broker.impl;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.broker.BrokerManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.broker.BrokerBasicVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.log.LogDirVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import org.apache.kafka.clients.admin.LogDirDescription;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class BrokerManagerImpl implements BrokerManager {
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<BrokerBasicVO> getBrokerBasic(Long clusterPhyId, Integer brokerId) {
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterPhyId));
|
|
||||||
}
|
|
||||||
|
|
||||||
Broker broker = brokerService.getBroker(clusterPhyId, brokerId);
|
|
||||||
if (broker == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getBrokerNotExist(clusterPhyId, brokerId));
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(new BrokerBasicVO(brokerId, broker.getHost(), clusterPhy.getName()));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<LogDirVO> getBrokerLogDirs(Long clusterPhyId, Integer brokerId, PaginationBaseDTO dto) {
|
|
||||||
Result<Map<String, LogDirDescription>> dirDescResult = brokerService.getBrokerLogDirDescFromKafka(clusterPhyId, brokerId);
|
|
||||||
if (dirDescResult.failed()) {
|
|
||||||
return PaginationResult.buildFailure(dirDescResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<String, LogDirDescription> dirDescMap = dirDescResult.hasData()? dirDescResult.getData(): new HashMap<>();
|
|
||||||
|
|
||||||
List<LogDirVO> voList = new ArrayList<>();
|
|
||||||
for (Map.Entry<String, LogDirDescription> entry: dirDescMap.entrySet()) {
|
|
||||||
entry.getValue().replicaInfos().entrySet().stream().forEach(elem -> {
|
|
||||||
LogDirVO vo = new LogDirVO();
|
|
||||||
vo.setDir(entry.getKey());
|
|
||||||
vo.setTopicName(elem.getKey().topic());
|
|
||||||
vo.setPartitionId(elem.getKey().partition());
|
|
||||||
vo.setOffsetLag(elem.getValue().offsetLag());
|
|
||||||
vo.setLogSizeUnitB(elem.getValue().size());
|
|
||||||
voList.add(vo);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return PaginationUtil.pageBySubData(
|
|
||||||
PaginationUtil.pageByFuzzyFilter(voList, dto.getSearchKeywords(), Arrays.asList("topicName")),
|
|
||||||
dto
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterBrokersOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.res.ClusterBrokersOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.res.ClusterBrokersStateVO;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 多集群总体状态
|
|
||||||
*/
|
|
||||||
public interface ClusterBrokersManager {
|
|
||||||
/**
|
|
||||||
* 获取缓存查询结果 & broker 表查询结果并集
|
|
||||||
* @param clusterPhyId kafka 物理集群 id
|
|
||||||
* @param dto 封装分页查询参数对象
|
|
||||||
* @return 返回获取到的缓存查询结果 & broker 表查询结果并集
|
|
||||||
*/
|
|
||||||
PaginationResult<ClusterBrokersOverviewVO> getClusterPhyBrokersOverview(Long clusterPhyId, ClusterBrokersOverviewDTO dto);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 根据物理集群id获取集群对应broker状态信息
|
|
||||||
* @param clusterPhyId 物理集群 id
|
|
||||||
* @return 返回根据物理集群id获取到的集群对应broker状态信息
|
|
||||||
*/
|
|
||||||
ClusterBrokersStateVO getClusterPhyBrokersState(Long clusterPhyId);
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterConnectorsOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connect.ConnectStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector.ClusterConnectorOverviewVO;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Kafka集群Connector概览
|
|
||||||
*/
|
|
||||||
public interface ClusterConnectorsManager {
|
|
||||||
PaginationResult<ClusterConnectorOverviewVO> getClusterConnectorsOverview(Long clusterPhyId, ClusterConnectorsOverviewDTO dto);
|
|
||||||
|
|
||||||
ConnectStateVO getClusterConnectorsState(Long clusterPhyId);
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterTopicsOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.res.ClusterPhyTopicsOverviewVO;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 多集群总体状态
|
|
||||||
*/
|
|
||||||
public interface ClusterTopicsManager {
|
|
||||||
PaginationResult<ClusterPhyTopicsOverviewVO> getClusterPhyTopicsOverview(Long clusterPhyId, ClusterTopicsOverviewDTO dto);
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterZookeepersOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.zookeeper.ClusterZookeepersOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.zookeeper.ClusterZookeepersStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.zookeeper.ZnodeVO;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 多集群总体状态
|
|
||||||
*/
|
|
||||||
public interface ClusterZookeepersManager {
|
|
||||||
Result<ClusterZookeepersStateVO> getClusterPhyZookeepersState(Long clusterPhyId);
|
|
||||||
|
|
||||||
PaginationResult<ClusterZookeepersOverviewVO> getClusterPhyZookeepersOverview(Long clusterPhyId, ClusterZookeepersOverviewDTO dto);
|
|
||||||
|
|
||||||
Result<ZnodeVO> getZnodeVO(Long clusterPhyId, String path);
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhysHealthState;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhysState;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.MultiClusterDashboardDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.ClusterPhyBaseVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.ClusterPhyDashboardVO;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 多集群总体状态
|
|
||||||
*/
|
|
||||||
public interface MultiClusterPhyManager {
|
|
||||||
/**
|
|
||||||
* 获取所有集群的状态
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
ClusterPhysState getClusterPhysState();
|
|
||||||
|
|
||||||
ClusterPhysHealthState getClusterPhysHealthState();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询多集群大盘
|
|
||||||
* @param dto 分页信息
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
PaginationResult<ClusterPhyDashboardVO> getClusterPhysDashboard(MultiClusterDashboardDTO dto);
|
|
||||||
|
|
||||||
Result<List<ClusterPhyBaseVO>> getClusterPhysBasic();
|
|
||||||
}
|
|
||||||
@@ -1,257 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.cluster.ClusterBrokersManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterBrokersOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.res.ClusterBrokersOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.res.ClusterBrokersStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.kafkacontroller.KafkaControllerVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.SortTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterRunStateEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationMetricsUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class ClusterBrokersManagerImpl implements ClusterBrokersManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(ClusterBrokersManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerConfigService brokerConfigService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerMetricService brokerMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KafkaControllerService kafkaControllerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KafkaJMXClient kafkaJMXClient;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<ClusterBrokersOverviewVO> getClusterPhyBrokersOverview(Long clusterPhyId, ClusterBrokersOverviewDTO dto) {
|
|
||||||
// 获取集群Broker列表
|
|
||||||
List<Broker> brokerList = brokerService.listAllBrokersFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
// 搜索
|
|
||||||
brokerList = PaginationUtil.pageByFuzzyFilter(brokerList, dto.getSearchKeywords(), Arrays.asList("host"));
|
|
||||||
|
|
||||||
// 获取指标
|
|
||||||
Result<List<BrokerMetrics>> metricsResult = brokerMetricService.getLatestMetricsFromES(
|
|
||||||
clusterPhyId,
|
|
||||||
brokerList.stream().filter(elem1 -> elem1.alive()).map(elem2 -> elem2.getBrokerId()).collect(Collectors.toList())
|
|
||||||
);
|
|
||||||
|
|
||||||
// 分页 + 搜索
|
|
||||||
PaginationResult<Integer> paginationResult = this.pagingBrokers(brokerList, metricsResult.hasData()? metricsResult.getData(): new ArrayList<>(), dto);
|
|
||||||
|
|
||||||
// 获取__consumer_offsetsTopic的分布
|
|
||||||
Topic groupTopic = topicService.getTopic(clusterPhyId, org.apache.kafka.common.internals.Topic.GROUP_METADATA_TOPIC_NAME);
|
|
||||||
Topic transactionTopic = topicService.getTopic(clusterPhyId, org.apache.kafka.common.internals.Topic.TRANSACTION_STATE_TOPIC_NAME);
|
|
||||||
|
|
||||||
//获取controller信息
|
|
||||||
KafkaController kafkaController = kafkaControllerService.getKafkaControllerFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
//获取jmx状态信息
|
|
||||||
Map<Integer, Boolean> jmxConnectedMap = new HashMap<>();
|
|
||||||
brokerList.forEach(elem -> jmxConnectedMap.put(elem.getBrokerId(), kafkaJMXClient.getClientWithCheck(clusterPhyId, elem.getBrokerId()) != null));
|
|
||||||
|
|
||||||
|
|
||||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
// 格式转换
|
|
||||||
return PaginationResult.buildSuc(
|
|
||||||
this.convert2ClusterBrokersOverviewVOList(
|
|
||||||
clusterPhy,
|
|
||||||
paginationResult.getData().getBizData(),
|
|
||||||
brokerList,
|
|
||||||
metricsResult.getData(),
|
|
||||||
groupTopic,
|
|
||||||
transactionTopic,
|
|
||||||
kafkaController,
|
|
||||||
jmxConnectedMap
|
|
||||||
),
|
|
||||||
paginationResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ClusterBrokersStateVO getClusterPhyBrokersState(Long clusterPhyId) {
|
|
||||||
ClusterBrokersStateVO clusterBrokersStateVO = new ClusterBrokersStateVO();
|
|
||||||
|
|
||||||
// 获取集群Broker列表
|
|
||||||
List<Broker> allBrokerList = brokerService.listAllBrokersFromDB(clusterPhyId);
|
|
||||||
if (allBrokerList == null) {
|
|
||||||
allBrokerList = new ArrayList<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 设置broker数
|
|
||||||
clusterBrokersStateVO.setBrokerCount(allBrokerList.size());
|
|
||||||
|
|
||||||
// 设置版本信息
|
|
||||||
clusterBrokersStateVO.setBrokerVersionList(
|
|
||||||
this.getBrokerVersionList(clusterPhyId, allBrokerList.stream().filter(elem -> elem.alive()).collect(Collectors.toList()))
|
|
||||||
);
|
|
||||||
|
|
||||||
// 获取controller信息
|
|
||||||
KafkaController kafkaController = kafkaControllerService.getKafkaControllerFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
// 设置kafka-controller信息
|
|
||||||
clusterBrokersStateVO.setKafkaControllerAlive(false);
|
|
||||||
if(null != kafkaController) {
|
|
||||||
clusterBrokersStateVO.setKafkaController(
|
|
||||||
this.convert2KafkaControllerVO(
|
|
||||||
kafkaController,
|
|
||||||
brokerService.getBroker(clusterPhyId, kafkaController.getBrokerId())
|
|
||||||
)
|
|
||||||
);
|
|
||||||
clusterBrokersStateVO.setKafkaControllerAlive(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
clusterBrokersStateVO.setConfigSimilar(brokerConfigService.countBrokerConfigDiffsFromDB(clusterPhyId, KafkaConstant.CONFIG_SIMILAR_IGNORED_CONFIG_KEY_LIST) <= 0
|
|
||||||
);
|
|
||||||
|
|
||||||
return clusterBrokersStateVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private PaginationResult<Integer> pagingBrokers(List<Broker> brokerList, List<BrokerMetrics> metricsList, PaginationSortDTO dto) {
|
|
||||||
if (ValidateUtils.isBlank(dto.getSortField())) {
|
|
||||||
// 默认排序
|
|
||||||
return PaginationUtil.pageBySubData(
|
|
||||||
PaginationUtil.pageBySort(brokerList, "brokerId", SortTypeEnum.ASC.getSortType()).stream().map(elem -> elem.getBrokerId()).collect(Collectors.toList()),
|
|
||||||
dto
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (!brokerMetricService.isMetricName(dto.getSortField())) {
|
|
||||||
// 非指标字段进行排序，分页
|
|
||||||
return PaginationUtil.pageBySubData(
|
|
||||||
PaginationUtil.pageBySort(brokerList, dto.getSortField(), dto.getSortType()).stream().map(elem -> elem.getBrokerId()).collect(Collectors.toList()),
|
|
||||||
dto
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 指标字段进行排序及分页
|
|
||||||
Map<Integer, BrokerMetrics> metricsMap = metricsList.stream().collect(Collectors.toMap(BrokerMetrics::getBrokerId, Function.identity()));
|
|
||||||
brokerList.stream().forEach(elem -> {
|
|
||||||
metricsMap.putIfAbsent(elem.getBrokerId(), new BrokerMetrics(elem.getClusterPhyId(), elem.getBrokerId()));
|
|
||||||
});
|
|
||||||
|
|
||||||
// 排序
|
|
||||||
metricsList = (List<BrokerMetrics>) PaginationMetricsUtil.sortMetrics(new ArrayList<>(metricsMap.values()), dto.getSortField(), "brokerId", dto.getSortType());
|
|
||||||
|
|
||||||
return PaginationUtil.pageBySubData(
|
|
||||||
metricsList.stream().map(elem -> elem.getBrokerId()).collect(Collectors.toList()),
|
|
||||||
dto
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<ClusterBrokersOverviewVO> convert2ClusterBrokersOverviewVOList(ClusterPhy clusterPhy,
|
|
||||||
List<Integer> pagedBrokerIdList,
|
|
||||||
List<Broker> brokerList,
|
|
||||||
List<BrokerMetrics> metricsList,
|
|
||||||
Topic groupTopic,
|
|
||||||
Topic transactionTopic,
|
|
||||||
KafkaController kafkaController,
|
|
||||||
Map<Integer, Boolean> jmxConnectedMap) {
|
|
||||||
Map<Integer, BrokerMetrics> metricsMap = metricsList == null ? new HashMap<>() : metricsList.stream().collect(Collectors.toMap(BrokerMetrics::getBrokerId, Function.identity()));
|
|
||||||
|
|
||||||
Map<Integer, Broker> brokerMap = brokerList == null ? new HashMap<>() : brokerList.stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
|
|
||||||
|
|
||||||
List<ClusterBrokersOverviewVO> voList = new ArrayList<>(pagedBrokerIdList.size());
|
|
||||||
for (Integer brokerId : pagedBrokerIdList) {
|
|
||||||
Broker broker = brokerMap.get(brokerId);
|
|
||||||
BrokerMetrics brokerMetrics = metricsMap.get(brokerId);
|
|
||||||
Boolean jmxConnected = jmxConnectedMap.get(brokerId);
|
|
||||||
voList.add(this.convert2ClusterBrokersOverviewVO(brokerId, broker, brokerMetrics, groupTopic, transactionTopic, kafkaController, jmxConnected));
|
|
||||||
}
|
|
||||||
|
|
||||||
//补充非zk模式的JMXPort信息
|
|
||||||
if (!clusterPhy.getRunState().equals(ClusterRunStateEnum.RUN_ZK.getRunState())) {
|
|
||||||
JmxConfig jmxConfig = ConvertUtil.str2ObjByJson(clusterPhy.getJmxProperties(), JmxConfig.class);
|
|
||||||
voList.forEach(elem -> elem.setJmxPort(jmxConfig.getFinallyJmxPort(String.valueOf(elem.getBrokerId()))));
|
|
||||||
}
|
|
||||||
|
|
||||||
return voList;
|
|
||||||
}
|
|
||||||
|
|
||||||
private ClusterBrokersOverviewVO convert2ClusterBrokersOverviewVO(Integer brokerId, Broker broker, BrokerMetrics brokerMetrics, Topic groupTopic, Topic transactionTopic, KafkaController kafkaController, Boolean jmxConnected) {
|
|
||||||
ClusterBrokersOverviewVO clusterBrokersOverviewVO = new ClusterBrokersOverviewVO();
|
|
||||||
clusterBrokersOverviewVO.setBrokerId(brokerId);
|
|
||||||
if (broker != null) {
|
|
||||||
clusterBrokersOverviewVO.setHost(broker.getHost());
|
|
||||||
clusterBrokersOverviewVO.setRack(broker.getRack());
|
|
||||||
clusterBrokersOverviewVO.setJmxPort(broker.getJmxPort());
|
|
||||||
clusterBrokersOverviewVO.setAlive(broker.alive());
|
|
||||||
clusterBrokersOverviewVO.setStartTimeUnitMs(broker.getStartTimestamp());
|
|
||||||
}
|
|
||||||
clusterBrokersOverviewVO.setKafkaRoleList(new ArrayList<>());
|
|
||||||
|
|
||||||
if (groupTopic != null && groupTopic.getBrokerIdSet().contains(brokerId)) {
|
|
||||||
clusterBrokersOverviewVO.getKafkaRoleList().add(groupTopic.getTopicName());
|
|
||||||
}
|
|
||||||
if (transactionTopic != null && transactionTopic.getBrokerIdSet().contains(brokerId)) {
|
|
||||||
clusterBrokersOverviewVO.getKafkaRoleList().add(transactionTopic.getTopicName());
|
|
||||||
}
|
|
||||||
if (kafkaController != null && kafkaController.getBrokerId().equals(brokerId)) {
|
|
||||||
clusterBrokersOverviewVO.getKafkaRoleList().add(KafkaConstant.CONTROLLER_ROLE);
|
|
||||||
}
|
|
||||||
|
|
||||||
clusterBrokersOverviewVO.setLatestMetrics(brokerMetrics);
|
|
||||||
clusterBrokersOverviewVO.setJmxConnected(jmxConnected);
|
|
||||||
return clusterBrokersOverviewVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
private KafkaControllerVO convert2KafkaControllerVO(KafkaController kafkaController, Broker kafkaControllerBroker) {
|
|
||||||
if(null != kafkaController && null != kafkaControllerBroker) {
|
|
||||||
KafkaControllerVO kafkaControllerVO = new KafkaControllerVO();
|
|
||||||
kafkaControllerVO.setBrokerId(kafkaController.getBrokerId());
|
|
||||||
kafkaControllerVO.setBrokerHost(kafkaControllerBroker.getHost());
|
|
||||||
return kafkaControllerVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<String> getBrokerVersionList(Long clusterPhyId, List<Broker> brokerList) {
|
|
||||||
Set<String> brokerVersionList = new HashSet<>();
|
|
||||||
for (Broker broker : brokerList) {
|
|
||||||
brokerVersionList.add(brokerService.getBrokerVersionFromKafkaWithCacheFirst(broker.getClusterPhyId(),broker.getBrokerId(),broker.getStartTimestamp()));
|
|
||||||
}
|
|
||||||
brokerVersionList.remove("");
|
|
||||||
return new ArrayList<>(brokerVersionList);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.cluster.ClusterConnectorsManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterConnectorsOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.connect.MetricsConnectorsDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectWorker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.WorkerConnector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectorMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.connect.ConnectorPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connect.ConnectStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector.ClusterConnectorOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.ConnectConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationMetricsUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerService;
|
|
||||||
import org.apache.kafka.connect.runtime.AbstractStatus;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class ClusterConnectorsManagerImpl implements ClusterConnectorsManager {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(ClusterConnectorsManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorService connectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectClusterService connectClusterService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorMetricService connectorMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private WorkerService workerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private WorkerConnectorService workerConnectorService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<ClusterConnectorOverviewVO> getClusterConnectorsOverview(Long clusterPhyId, ClusterConnectorsOverviewDTO dto) {
|
|
||||||
List<ConnectCluster> clusterList = connectClusterService.listByKafkaCluster(clusterPhyId);
|
|
||||||
|
|
||||||
List<ConnectorPO> poList = connectorService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
// 查询实时指标
|
|
||||||
Result<List<ConnectorMetrics>> latestMetricsResult = connectorMetricService.getLatestMetricsFromES(
|
|
||||||
clusterPhyId,
|
|
||||||
poList.stream().map(elem -> new ClusterConnectorDTO(elem.getConnectClusterId(), elem.getConnectorName())).collect(Collectors.toList()),
|
|
||||||
dto.getLatestMetricNames()
|
|
||||||
);
|
|
||||||
|
|
||||||
if (latestMetricsResult.failed()) {
|
|
||||||
LOGGER.error("method=getClusterConnectorsOverview||clusterPhyId={}||result={}||errMsg=get latest metric failed", clusterPhyId, latestMetricsResult);
|
|
||||||
return PaginationResult.buildFailure(latestMetricsResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 转换成vo
|
|
||||||
List<ClusterConnectorOverviewVO> voList = ConnectConverter.convert2ClusterConnectorOverviewVOList(clusterList, poList,latestMetricsResult.getData());
|
|
||||||
|
|
||||||
// 请求分页信息
|
|
||||||
PaginationResult<ClusterConnectorOverviewVO> voPaginationResult = this.pagingConnectorInLocal(voList, dto);
|
|
||||||
if (voPaginationResult.failed()) {
|
|
||||||
LOGGER.error("method=getClusterConnectorsOverview||clusterPhyId={}||result={}||errMsg=pagination in local failed", clusterPhyId, voPaginationResult);
|
|
||||||
|
|
||||||
return PaginationResult.buildFailure(voPaginationResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 查询历史指标
|
|
||||||
Result<List<MetricMultiLinesVO>> lineMetricsResult = connectorMetricService.listConnectClusterMetricsFromES(
|
|
||||||
clusterPhyId,
|
|
||||||
this.buildMetricsConnectorsDTO(
|
|
||||||
voPaginationResult.getData().getBizData().stream().map(elem -> new ClusterConnectorDTO(elem.getConnectClusterId(), elem.getConnectorName())).collect(Collectors.toList()),
|
|
||||||
dto.getMetricLines()
|
|
||||||
)
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(
|
|
||||||
ConnectConverter.supplyData2ClusterConnectorOverviewVOList(
|
|
||||||
voPaginationResult.getData().getBizData(),
|
|
||||||
lineMetricsResult.getData()
|
|
||||||
),
|
|
||||||
voPaginationResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ConnectStateVO getClusterConnectorsState(Long clusterPhyId) {
|
|
||||||
//获取Connect集群Id列表
|
|
||||||
List<ConnectCluster> connectClusterList = connectClusterService.listByKafkaCluster(clusterPhyId);
|
|
||||||
List<ConnectorPO> connectorPOList = connectorService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
List<WorkerConnector> workerConnectorList = workerConnectorService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
List<ConnectWorker> connectWorkerList = workerService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
return convert2ConnectStateVO(connectClusterList, connectorPOList, workerConnectorList, connectWorkerList);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private MetricsConnectorsDTO buildMetricsConnectorsDTO(List<ClusterConnectorDTO> connectorDTOList, MetricDTO metricDTO) {
|
|
||||||
MetricsConnectorsDTO dto = ConvertUtil.obj2Obj(metricDTO, MetricsConnectorsDTO.class);
|
|
||||||
dto.setConnectorNameList(connectorDTOList == null? new ArrayList<>(): connectorDTOList);
|
|
||||||
|
|
||||||
return dto;
|
|
||||||
}
|
|
||||||
|
|
||||||
private ConnectStateVO convert2ConnectStateVO(List<ConnectCluster> connectClusterList, List<ConnectorPO> connectorPOList, List<WorkerConnector> workerConnectorList, List<ConnectWorker> connectWorkerList) {
|
|
||||||
ConnectStateVO connectStateVO = new ConnectStateVO();
|
|
||||||
connectStateVO.setConnectClusterCount(connectClusterList.size());
|
|
||||||
connectStateVO.setTotalConnectorCount(connectorPOList.size());
|
|
||||||
connectStateVO.setAliveConnectorCount(connectorPOList.stream().filter(elem -> elem.getState().equals(AbstractStatus.State.RUNNING.name())).collect(Collectors.toList()).size());
|
|
||||||
connectStateVO.setWorkerCount(connectWorkerList.size());
|
|
||||||
connectStateVO.setTotalTaskCount(workerConnectorList.size());
|
|
||||||
connectStateVO.setAliveTaskCount(workerConnectorList.stream().filter(elem -> elem.getState().equals(AbstractStatus.State.RUNNING.name())).collect(Collectors.toList()).size());
|
|
||||||
return connectStateVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
private PaginationResult<ClusterConnectorOverviewVO> pagingConnectorInLocal(List<ClusterConnectorOverviewVO> connectorVOList, ClusterConnectorsOverviewDTO dto) {
|
|
||||||
//模糊匹配
|
|
||||||
connectorVOList = PaginationUtil.pageByFuzzyFilter(connectorVOList, dto.getSearchKeywords(), Arrays.asList("connectorName"));
|
|
||||||
|
|
||||||
//排序
|
|
||||||
if (!dto.getLatestMetricNames().isEmpty()) {
|
|
||||||
PaginationMetricsUtil.sortMetrics(connectorVOList, "latestMetrics", dto.getSortMetricNameList(), "connectorName", dto.getSortType());
|
|
||||||
} else {
|
|
||||||
PaginationUtil.pageBySort(connectorVOList, dto.getSortField(), dto.getSortType(), "connectorName", dto.getSortType());
|
|
||||||
}
|
|
||||||
|
|
||||||
//分页
|
|
||||||
return PaginationUtil.pageBySubData(connectorVOList, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,120 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.cluster.ClusterTopicsManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterTopicsOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsTopicDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.res.ClusterPhyTopicsOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.TopicVOConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.ha.HaResTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationMetricsUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.ha.HaActiveStandbyRelationService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class ClusterTopicsManagerImpl implements ClusterTopicsManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(ClusterTopicsManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicMetricService topicMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private HaActiveStandbyRelationService haActiveStandbyRelationService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<ClusterPhyTopicsOverviewVO> getClusterPhyTopicsOverview(Long clusterPhyId, ClusterTopicsOverviewDTO dto) {
|
|
||||||
// 获取集群所有的Topic信息
|
|
||||||
List<Topic> topicList = topicService.listTopicsFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
// 获取集群所有Topic的指标
|
|
||||||
Map<String, TopicMetrics> metricsMap = topicMetricService.getLatestMetricsFromCache(clusterPhyId);
|
|
||||||
|
|
||||||
// 获取HA信息
|
|
||||||
Set<String> haTopicNameSet = haActiveStandbyRelationService.listByClusterAndType(clusterPhyId, HaResTypeEnum.MIRROR_TOPIC).stream().map(elem -> elem.getResName()).collect(Collectors.toSet());
|
|
||||||
|
|
||||||
// 转换成vo
|
|
||||||
List<ClusterPhyTopicsOverviewVO> voList = TopicVOConverter.convert2ClusterPhyTopicsOverviewVOList(topicList, metricsMap, haTopicNameSet);
|
|
||||||
|
|
||||||
// 请求分页信息
|
|
||||||
PaginationResult<ClusterPhyTopicsOverviewVO> voPaginationResult = this.pagingTopicInLocal(voList, dto);
|
|
||||||
if (voPaginationResult.failed()) {
|
|
||||||
log.error("method=getClusterPhyTopicsOverview||clusterPhyId={}||result={}||errMsg=pagination in local failed", clusterPhyId, voPaginationResult);
|
|
||||||
|
|
||||||
return PaginationResult.buildFailure(voPaginationResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 查询指标
|
|
||||||
Result<List<MetricMultiLinesVO>> metricMultiLinesResult = topicMetricService.listTopicMetricsFromES(
|
|
||||||
clusterPhyId,
|
|
||||||
this.buildTopicOverviewMetricsDTO(voPaginationResult.getData().getBizData().stream().map(elem -> elem.getTopicName()).collect(Collectors.toList()), dto.getMetricLines())
|
|
||||||
);
|
|
||||||
|
|
||||||
if (metricMultiLinesResult.failed()) {
|
|
||||||
// 查询ES失败或者ES无数据，则ES可能存在问题，此时降级返回Topic的基本信息数据
|
|
||||||
log.error("method=getClusterPhyTopicsOverview||clusterPhyId={}||result={}||errMsg=get metrics from es failed", clusterPhyId, metricMultiLinesResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(
|
|
||||||
TopicVOConverter.supplyMetricLines(
|
|
||||||
voPaginationResult.getData().getBizData(),
|
|
||||||
metricMultiLinesResult.getData() == null? new ArrayList<>(): metricMultiLinesResult.getData()
|
|
||||||
),
|
|
||||||
voPaginationResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private MetricsTopicDTO buildTopicOverviewMetricsDTO(List<String> topicNameList, MetricDTO metricDTO) {
|
|
||||||
MetricsTopicDTO dto = ConvertUtil.obj2Obj(metricDTO, MetricsTopicDTO.class);
|
|
||||||
dto.setTopics(topicNameList == null? new ArrayList<>(): topicNameList);
|
|
||||||
return dto;
|
|
||||||
}
|
|
||||||
|
|
||||||
private PaginationResult<ClusterPhyTopicsOverviewVO> pagingTopicInLocal(List<ClusterPhyTopicsOverviewVO> voList, ClusterTopicsOverviewDTO dto) {
|
|
||||||
List<ClusterPhyTopicsOverviewVO> metricsList = voList.stream().filter(elem -> {
|
|
||||||
if (dto.getShowInternalTopics() != null && dto.getShowInternalTopics()) {
|
|
||||||
// 仅展示系统Topic
|
|
||||||
return KafkaConstant.KAFKA_INTERNAL_TOPICS.contains(elem.getTopicName());
|
|
||||||
} else {
|
|
||||||
// 仅展示用户Topic
|
|
||||||
return !KafkaConstant.KAFKA_INTERNAL_TOPICS.contains(elem.getTopicName());
|
|
||||||
}
|
|
||||||
}).collect(Collectors.toList());
|
|
||||||
|
|
||||||
// 名称搜索
|
|
||||||
metricsList = PaginationUtil.pageByFuzzyFilter(metricsList, dto.getSearchKeywords(), Arrays.asList("topicName"));
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(dto.getSortField()) && !"createTime".equals(dto.getSortField())) {
|
|
||||||
// 指标排序
|
|
||||||
PaginationMetricsUtil.sortMetrics(metricsList, "latestMetrics", dto.getSortField(), "topicName", dto.getSortType());
|
|
||||||
} else {
|
|
||||||
// 信息排序
|
|
||||||
PaginationUtil.pageBySort(metricsList, dto.getSortField(), dto.getSortType(), "topicName", dto.getSortType());
|
|
||||||
}
|
|
||||||
|
|
||||||
return PaginationUtil.pageBySubData(metricsList, dto);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.cluster.ClusterZookeepersManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterZookeepersOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ZookeeperMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.zookeeper.Znode;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.zookeeper.ZookeeperInfo;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.zookeeper.ClusterZookeepersOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.zookeeper.ClusterZookeepersStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.zookeeper.ZnodeVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.zookeeper.ZKRoleEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.ZookeeperMetricVersionItems;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.zookeeper.ZnodeService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.zookeeper.ZookeeperMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.zookeeper.ZookeeperService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class ClusterZookeepersManagerImpl implements ClusterZookeepersManager {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(ClusterZookeepersManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ZookeeperService zookeeperService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ZookeeperMetricService zookeeperMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ZnodeService znodeService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<ClusterZookeepersStateVO> getClusterPhyZookeepersState(Long clusterPhyId) {
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterPhyId));
|
|
||||||
}
|
|
||||||
|
|
||||||
List<ZookeeperInfo> infoList = zookeeperService.listFromDBByCluster(clusterPhyId);
|
|
||||||
|
|
||||||
ClusterZookeepersStateVO vo = new ClusterZookeepersStateVO();
|
|
||||||
vo.setTotalServerCount(infoList.size());
|
|
||||||
vo.setAliveFollowerCount(0);
|
|
||||||
vo.setTotalFollowerCount(0);
|
|
||||||
vo.setAliveObserverCount(0);
|
|
||||||
vo.setTotalObserverCount(0);
|
|
||||||
vo.setAliveServerCount(0);
|
|
||||||
for (ZookeeperInfo info: infoList) {
|
|
||||||
if (info.getRole().equals(ZKRoleEnum.LEADER.getRole()) || info.getRole().equals(ZKRoleEnum.STANDALONE.getRole())) {
|
|
||||||
// leader 或者 standalone
|
|
||||||
vo.setLeaderNode(info.getHost());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (info.getRole().equals(ZKRoleEnum.FOLLOWER.getRole())) {
|
|
||||||
vo.setTotalFollowerCount(vo.getTotalFollowerCount() + 1);
|
|
||||||
vo.setAliveFollowerCount(info.alive()? vo.getAliveFollowerCount() + 1: vo.getAliveFollowerCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (info.getRole().equals(ZKRoleEnum.OBSERVER.getRole())) {
|
|
||||||
vo.setTotalObserverCount(vo.getTotalObserverCount() + 1);
|
|
||||||
vo.setAliveObserverCount(info.alive()? vo.getAliveObserverCount() + 1: vo.getAliveObserverCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (info.alive()) {
|
|
||||||
vo.setAliveServerCount(vo.getAliveServerCount() + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 指标获取
|
|
||||||
Result<ZookeeperMetrics> metricsResult = zookeeperMetricService.batchCollectMetricsFromZookeeper(
|
|
||||||
clusterPhyId,
|
|
||||||
Arrays.asList(
|
|
||||||
ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_WATCH_COUNT,
|
|
||||||
ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_HEALTH_STATE,
|
|
||||||
ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_HEALTH_CHECK_PASSED,
|
|
||||||
ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_HEALTH_CHECK_TOTAL
|
|
||||||
)
|
|
||||||
|
|
||||||
);
|
|
||||||
if (metricsResult.failed()) {
|
|
||||||
LOGGER.error(
|
|
||||||
"method=getClusterPhyZookeepersState||clusterPhyId={}||errMsg={}",
|
|
||||||
clusterPhyId, metricsResult.getMessage()
|
|
||||||
);
|
|
||||||
return Result.buildSuc(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
ZookeeperMetrics metrics = metricsResult.getData();
|
|
||||||
vo.setWatchCount(ConvertUtil.float2Integer(metrics.getMetrics().get(ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_WATCH_COUNT)));
|
|
||||||
vo.setHealthState(ConvertUtil.float2Integer(metrics.getMetrics().get(ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_HEALTH_STATE)));
|
|
||||||
vo.setHealthCheckPassed(ConvertUtil.float2Integer(metrics.getMetrics().get(ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_HEALTH_CHECK_PASSED)));
|
|
||||||
vo.setHealthCheckTotal(ConvertUtil.float2Integer(metrics.getMetrics().get(ZookeeperMetricVersionItems.ZOOKEEPER_METRIC_HEALTH_CHECK_TOTAL)));
|
|
||||||
|
|
||||||
return Result.buildSuc(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<ClusterZookeepersOverviewVO> getClusterPhyZookeepersOverview(Long clusterPhyId, ClusterZookeepersOverviewDTO dto) {
|
|
||||||
//获取集群zookeeper列表
|
|
||||||
List<ClusterZookeepersOverviewVO> clusterZookeepersOverviewVOList = ConvertUtil.list2List(zookeeperService.listFromDBByCluster(clusterPhyId), ClusterZookeepersOverviewVO.class);
|
|
||||||
|
|
||||||
//搜索
|
|
||||||
clusterZookeepersOverviewVOList = PaginationUtil.pageByFuzzyFilter(clusterZookeepersOverviewVOList, dto.getSearchKeywords(), Arrays.asList("host"));
|
|
||||||
|
|
||||||
//分页
|
|
||||||
PaginationResult<ClusterZookeepersOverviewVO> paginationResult = PaginationUtil.pageBySubData(clusterZookeepersOverviewVOList, dto);
|
|
||||||
|
|
||||||
return paginationResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<ZnodeVO> getZnodeVO(Long clusterPhyId, String path) {
|
|
||||||
Result<Znode> result = znodeService.getZnode(clusterPhyId, path);
|
|
||||||
if (result.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(result);
|
|
||||||
}
|
|
||||||
return Result.buildSuc(ConvertUtil.obj2ObjByJSON(result.getData(), ZnodeVO.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,207 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.cluster.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.cluster.MultiClusterPhyManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsClusterPhyDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhysHealthState;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhysState;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.MultiClusterDashboardDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.ClusterPhyBaseVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.ClusterPhyDashboardVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.ClusterVOConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthStateEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationMetricsUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.ClusterMetricVersionItems;
|
|
||||||
import com.xiaojukeji.know.streaming.km.rebalance.algorithm.model.Resource;
|
|
||||||
import com.xiaojukeji.know.streaming.km.rebalance.common.BalanceMetricConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.rebalance.common.bean.entity.ClusterBalanceItemState;
|
|
||||||
import com.xiaojukeji.know.streaming.km.rebalance.core.service.ClusterBalanceService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class MultiClusterPhyManagerImpl implements MultiClusterPhyManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(MultiClusterPhyManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterMetricService clusterMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterBalanceService clusterBalanceService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ClusterPhysState getClusterPhysState() {
|
|
||||||
List<ClusterPhy> clusterPhyList = clusterPhyService.listAllClusters();
|
|
||||||
ClusterPhysState physState = new ClusterPhysState(0, 0, 0, clusterPhyList.size());
|
|
||||||
|
|
||||||
for (ClusterPhy clusterPhy : clusterPhyList) {
|
|
||||||
ClusterMetrics metrics = clusterMetricService.getLatestMetricsFromCache(clusterPhy.getId());
|
|
||||||
Float state = metrics.getMetric(ClusterMetricVersionItems.CLUSTER_METRIC_HEALTH_STATE);
|
|
||||||
if (state == null) {
|
|
||||||
physState.setUnknownCount(physState.getUnknownCount() + 1);
|
|
||||||
} else if (state.intValue() == HealthStateEnum.DEAD.getDimension()) {
|
|
||||||
physState.setDownCount(physState.getDownCount() + 1);
|
|
||||||
} else {
|
|
||||||
physState.setLiveCount(physState.getLiveCount() + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return physState;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ClusterPhysHealthState getClusterPhysHealthState() {
|
|
||||||
List<ClusterPhy> clusterPhyList = clusterPhyService.listAllClusters();
|
|
||||||
|
|
||||||
ClusterPhysHealthState physState = new ClusterPhysHealthState(clusterPhyList.size());
|
|
||||||
for (ClusterPhy clusterPhy: clusterPhyList) {
|
|
||||||
ClusterMetrics metrics = clusterMetricService.getLatestMetricsFromCache(clusterPhy.getId());
|
|
||||||
Float state = metrics.getMetric(ClusterMetricVersionItems.CLUSTER_METRIC_HEALTH_STATE);
|
|
||||||
if (state == null) {
|
|
||||||
physState.setUnknownCount(physState.getUnknownCount() + 1);
|
|
||||||
} else if (state.intValue() == HealthStateEnum.GOOD.getDimension()) {
|
|
||||||
physState.setGoodCount(physState.getGoodCount() + 1);
|
|
||||||
} else if (state.intValue() == HealthStateEnum.MEDIUM.getDimension()) {
|
|
||||||
physState.setMediumCount(physState.getMediumCount() + 1);
|
|
||||||
} else if (state.intValue() == HealthStateEnum.POOR.getDimension()) {
|
|
||||||
physState.setPoorCount(physState.getPoorCount() + 1);
|
|
||||||
} else if (state.intValue() == HealthStateEnum.DEAD.getDimension()) {
|
|
||||||
physState.setDeadCount(physState.getDeadCount() + 1);
|
|
||||||
} else {
|
|
||||||
physState.setUnknownCount(physState.getUnknownCount() + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return physState;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<ClusterPhyDashboardVO> getClusterPhysDashboard(MultiClusterDashboardDTO dto) {
|
|
||||||
// 获取集群
|
|
||||||
List<ClusterPhy> clusterPhyList = clusterPhyService.listAllClusters();
|
|
||||||
|
|
||||||
// 转为vo格式，方便后续进行分页筛选等
|
|
||||||
List<ClusterPhyDashboardVO> voList = ConvertUtil.list2List(clusterPhyList, ClusterPhyDashboardVO.class);
|
|
||||||
|
|
||||||
// 本地分页过滤
|
|
||||||
voList = this.getAndPagingDataInLocal(voList, dto);
|
|
||||||
|
|
||||||
// ES分页过滤
|
|
||||||
PaginationResult<ClusterMetrics> latestMetricsResult = this.getAndPagingClusterWithLatestMetricsFromCache(voList, dto);
|
|
||||||
if (latestMetricsResult.failed()) {
|
|
||||||
log.error("method=getClusterPhysDashboard||pagingData={}||result={}||errMsg=search es data failed.", dto, latestMetricsResult);
|
|
||||||
return PaginationResult.buildFailure(latestMetricsResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 获取历史指标
|
|
||||||
Result<List<MetricMultiLinesVO>> linesMetricResult = clusterMetricService.listClusterMetricsFromES(
|
|
||||||
this.buildMetricsClusterPhyDTO(
|
|
||||||
latestMetricsResult.getData().getBizData().stream().map(elem -> elem.getClusterPhyId()).collect(Collectors.toList()),
|
|
||||||
dto.getMetricLines()
|
|
||||||
));
|
|
||||||
|
|
||||||
// 组装最终数据
|
|
||||||
return PaginationResult.buildSuc(
|
|
||||||
ClusterVOConverter.convert2ClusterPhyDashboardVOList(voList, linesMetricResult.getData(), latestMetricsResult.getData().getBizData()),
|
|
||||||
latestMetricsResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<ClusterPhyBaseVO>> getClusterPhysBasic() {
|
|
||||||
// 获取集群
|
|
||||||
List<ClusterPhy> clusterPhyList = clusterPhyService.listAllClusters();
|
|
||||||
|
|
||||||
// 转为vo格式，方便后续进行分页筛选等
|
|
||||||
return Result.buildSuc(ConvertUtil.list2List(clusterPhyList, ClusterPhyBaseVO.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
|
|
||||||
private List<ClusterPhyDashboardVO> getAndPagingDataInLocal(List<ClusterPhyDashboardVO> voList, MultiClusterDashboardDTO dto) {
|
|
||||||
// 时间排序
|
|
||||||
if ("createTime".equals(dto.getSortField())) {
|
|
||||||
voList = PaginationUtil.pageBySort(voList, "createTime", dto.getSortType(), "name", dto.getSortType());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 名称搜索
|
|
||||||
if (!ValidateUtils.isBlank(dto.getSearchKeywords())) {
|
|
||||||
voList = PaginationUtil.pageByFuzzyFilter(voList, dto.getSearchKeywords(), Arrays.asList("name"));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 精确搜索
|
|
||||||
return PaginationUtil.pageByPreciseFilter(voList, dto.getPreciseFilterDTOList());
|
|
||||||
}
|
|
||||||
|
|
||||||
private PaginationResult<ClusterMetrics> getAndPagingClusterWithLatestMetricsFromCache(List<ClusterPhyDashboardVO> voList, MultiClusterDashboardDTO dto) {
|
|
||||||
// 获取所有的metrics
|
|
||||||
List<ClusterMetrics> metricsList = new ArrayList<>();
|
|
||||||
for (ClusterPhyDashboardVO vo: voList) {
|
|
||||||
ClusterMetrics clusterMetrics = clusterMetricService.getLatestMetricsFromCache(vo.getId());
|
|
||||||
clusterMetrics.getMetrics().putIfAbsent(ClusterMetricVersionItems.CLUSTER_METRIC_HEALTH_STATE, (float) HealthStateEnum.UNKNOWN.getDimension());
|
|
||||||
|
|
||||||
Result<ClusterMetrics> balanceMetricsResult = this.getClusterLoadReBalanceInfo(vo.getId());
|
|
||||||
if (balanceMetricsResult.hasData()) {
|
|
||||||
clusterMetrics.putMetric(balanceMetricsResult.getData().getMetrics());
|
|
||||||
}
|
|
||||||
|
|
||||||
metricsList.add(clusterMetrics);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 范围搜索
|
|
||||||
metricsList = (List<ClusterMetrics>) PaginationMetricsUtil.rangeFilterMetrics(metricsList, dto.getRangeFilterDTOList());
|
|
||||||
|
|
||||||
// 精确搜索
|
|
||||||
metricsList = (List<ClusterMetrics>) PaginationMetricsUtil.preciseFilterMetrics(metricsList, dto.getPreciseFilterDTOList());
|
|
||||||
|
|
||||||
// 排序
|
|
||||||
PaginationMetricsUtil.sortMetrics(metricsList, dto.getSortField(), "clusterPhyId", dto.getSortType());
|
|
||||||
|
|
||||||
// 分页
|
|
||||||
return PaginationUtil.pageBySubData(metricsList, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
private MetricsClusterPhyDTO buildMetricsClusterPhyDTO(List<Long> clusterIdList, MetricDTO metricDTO) {
|
|
||||||
MetricsClusterPhyDTO dto = ConvertUtil.obj2Obj(metricDTO, MetricsClusterPhyDTO.class);
|
|
||||||
dto.setClusterPhyIds(clusterIdList);
|
|
||||||
return dto;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Result<ClusterMetrics> getClusterLoadReBalanceInfo(Long clusterPhyId) {
|
|
||||||
Result<ClusterBalanceItemState> stateResult = clusterBalanceService.getItemStateFromCacheFirst(clusterPhyId);
|
|
||||||
if (stateResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(stateResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
ClusterBalanceItemState state = stateResult.getData();
|
|
||||||
|
|
||||||
ClusterMetrics metric = ClusterMetrics.initWithMetrics(clusterPhyId, BalanceMetricConstant.CLUSTER_METRIC_LOAD_RE_BALANCE_ENABLE, state.getEnable()? Constant.YES: Constant.NO);
|
|
||||||
metric.putMetric(BalanceMetricConstant.CLUSTER_METRIC_LOAD_RE_BALANCE_CPU, state.getResItemState(Resource.CPU).floatValue());
|
|
||||||
metric.putMetric(BalanceMetricConstant.CLUSTER_METRIC_LOAD_RE_BALANCE_NW_IN, state.getResItemState(Resource.NW_IN).floatValue());
|
|
||||||
metric.putMetric(BalanceMetricConstant.CLUSTER_METRIC_LOAD_RE_BALANCE_NW_OUT, state.getResItemState(Resource.NW_OUT).floatValue());
|
|
||||||
metric.putMetric(BalanceMetricConstant.CLUSTER_METRIC_LOAD_RE_BALANCE_DISK, state.getResItemState(Resource.DISK).floatValue());
|
|
||||||
|
|
||||||
return Result.buildSuc(metric);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.connect.connector;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector.ConnectorCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.connector.ConnectorStateVO;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
public interface ConnectorManager {
|
|
||||||
Result<Void> updateConnectorConfig(Long connectClusterId, String connectorName, Properties configs, String operator);
|
|
||||||
|
|
||||||
Result<Void> createConnector(ConnectorCreateDTO dto, String operator);
|
|
||||||
Result<Void> createConnector(ConnectorCreateDTO dto, String heartbeatName, String checkpointName, String operator);
|
|
||||||
|
|
||||||
Result<ConnectorStateVO> getConnectorStateVO(Long connectClusterId, String connectorName);
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.connect.connector;
|
|
||||||
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.task.KCTaskOverviewVO;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/11/14
|
|
||||||
*/
|
|
||||||
public interface WorkerConnectorManager {
|
|
||||||
Result<List<KCTaskOverviewVO>> getTaskOverview(Long connectClusterId, String connectorName);
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.connect.connector.impl;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.connect.connector.ConnectorManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector.ConnectorCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.WorkerConnector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.config.ConnectConfigInfos;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector.KSConnector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector.KSConnectorInfo;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.connect.ConnectorPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.connector.ConnectorStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.connect.KafkaConnectConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.OpConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.plugin.PluginService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerConnectorService;
|
|
||||||
import org.apache.kafka.connect.runtime.AbstractStatus;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class ConnectorManagerImpl implements ConnectorManager {
|
|
||||||
@Autowired
|
|
||||||
private PluginService pluginService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorService connectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private OpConnectorService opConnectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private WorkerConnectorService workerConnectorService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> updateConnectorConfig(Long connectClusterId, String connectorName, Properties configs, String operator) {
|
|
||||||
Result<ConnectConfigInfos> infosResult = pluginService.validateConfig(connectClusterId, configs);
|
|
||||||
if (infosResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(infosResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (infosResult.getData().getErrorCount() > 0) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "Connector参数错误");
|
|
||||||
}
|
|
||||||
|
|
||||||
return opConnectorService.updateConnectorConfig(connectClusterId, connectorName, configs, operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> createConnector(ConnectorCreateDTO dto, String operator) {
|
|
||||||
dto.getSuitableConfig().put(KafkaConnectConstant.MIRROR_MAKER_NAME_FIELD_NAME, dto.getConnectorName());
|
|
||||||
|
|
||||||
Result<KSConnectorInfo> createResult = opConnectorService.createConnector(dto.getConnectClusterId(), dto.getConnectorName(), dto.getSuitableConfig(), operator);
|
|
||||||
if (createResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(createResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<KSConnector> ksConnectorResult = connectorService.getConnectorFromKafka(dto.getConnectClusterId(), dto.getConnectorName());
|
|
||||||
if (ksConnectorResult.failed()) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.SUCCESS, "创建成功，但是获取元信息失败，页面元信息会存在1分钟延迟");
|
|
||||||
}
|
|
||||||
|
|
||||||
opConnectorService.addNewToDB(ksConnectorResult.getData());
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> createConnector(ConnectorCreateDTO dto, String heartbeatName, String checkpointName, String operator) {
|
|
||||||
dto.getSuitableConfig().put(KafkaConnectConstant.MIRROR_MAKER_NAME_FIELD_NAME, dto.getConnectorName());
|
|
||||||
|
|
||||||
Result<KSConnectorInfo> createResult = opConnectorService.createConnector(dto.getConnectClusterId(), dto.getConnectorName(), dto.getSuitableConfig(), operator);
|
|
||||||
if (createResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(createResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<KSConnector> ksConnectorResult = connectorService.getConnectorFromKafka(dto.getConnectClusterId(), dto.getConnectorName());
|
|
||||||
if (ksConnectorResult.failed()) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.SUCCESS, "创建成功，但是获取元信息失败，页面元信息会存在1分钟延迟");
|
|
||||||
}
|
|
||||||
|
|
||||||
KSConnector connector = ksConnectorResult.getData();
|
|
||||||
connector.setCheckpointConnectorName(checkpointName);
|
|
||||||
connector.setHeartbeatConnectorName(heartbeatName);
|
|
||||||
|
|
||||||
opConnectorService.addNewToDB(connector);
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<ConnectorStateVO> getConnectorStateVO(Long connectClusterId, String connectorName) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, connectorName);
|
|
||||||
|
|
||||||
if (connectorPO == null) {
|
|
||||||
return Result.buildFailure(ResultStatus.NOT_EXIST);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<WorkerConnector> workerConnectorList = workerConnectorService.listFromDB(connectClusterId).stream().filter(elem -> elem.getConnectorName().equals(connectorName)).collect(Collectors.toList());
|
|
||||||
|
|
||||||
return Result.buildSuc(convert2ConnectorOverviewVO(connectorPO, workerConnectorList));
|
|
||||||
}
|
|
||||||
|
|
||||||
private ConnectorStateVO convert2ConnectorOverviewVO(ConnectorPO connectorPO, List<WorkerConnector> workerConnectorList) {
|
|
||||||
ConnectorStateVO connectorStateVO = new ConnectorStateVO();
|
|
||||||
connectorStateVO.setConnectClusterId(connectorPO.getConnectClusterId());
|
|
||||||
connectorStateVO.setName(connectorPO.getConnectorName());
|
|
||||||
connectorStateVO.setType(connectorPO.getConnectorType());
|
|
||||||
connectorStateVO.setState(connectorPO.getState());
|
|
||||||
connectorStateVO.setTotalTaskCount(workerConnectorList.size());
|
|
||||||
connectorStateVO.setAliveTaskCount(workerConnectorList.stream().filter(elem -> elem.getState().equals(AbstractStatus.State.RUNNING.name())).collect(Collectors.toList()).size());
|
|
||||||
connectorStateVO.setTotalWorkerCount(workerConnectorList.stream().map(elem -> elem.getWorkerId()).collect(Collectors.toSet()).size());
|
|
||||||
return connectorStateVO;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.connect.connector.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.connect.connector.WorkerConnectorManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.WorkerConnector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.task.KCTaskOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.connect.cache.LoadedConnectClusterCache;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/11/14
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class WorkerConnectorManageImpl implements WorkerConnectorManager {
|
|
||||||
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(WorkerConnectorManageImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private WorkerConnectorService workerConnectorService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<KCTaskOverviewVO>> getTaskOverview(Long connectClusterId, String connectorName) {
|
|
||||||
ConnectCluster connectCluster = LoadedConnectClusterCache.getByPhyId(connectClusterId);
|
|
||||||
List<WorkerConnector> workerConnectorList = workerConnectorService.getWorkerConnectorListFromCluster(connectCluster, connectorName);
|
|
||||||
|
|
||||||
return Result.buildSuc(ConvertUtil.list2List(workerConnectorList, KCTaskOverviewVO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.connect.mm2;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterMirrorMakersOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.mm2.MirrorMakerCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.mm2.ClusterMirrorMakerOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.mm2.MirrorMakerBaseStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.mm2.MirrorMakerStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin.ConnectConfigInfosVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.task.KCTaskOverviewVO;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/12/26
|
|
||||||
*/
|
|
||||||
public interface MirrorMakerManager {
|
|
||||||
Result<Void> createMirrorMaker(MirrorMakerCreateDTO dto, String operator);
|
|
||||||
|
|
||||||
Result<Void> deleteMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator);
|
|
||||||
|
|
||||||
Result<Void> modifyMirrorMakerConfig(MirrorMakerCreateDTO dto, String operator);
|
|
||||||
|
|
||||||
Result<Void> restartMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator);
|
|
||||||
Result<Void> stopMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator);
|
|
||||||
Result<Void> resumeMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator);
|
|
||||||
|
|
||||||
Result<MirrorMakerStateVO> getMirrorMakerStateVO(Long clusterPhyId);
|
|
||||||
|
|
||||||
PaginationResult<ClusterMirrorMakerOverviewVO> getClusterMirrorMakersOverview(Long clusterPhyId, ClusterMirrorMakersOverviewDTO dto);
|
|
||||||
|
|
||||||
|
|
||||||
Result<MirrorMakerBaseStateVO> getMirrorMakerState(Long connectId, String connectName);
|
|
||||||
|
|
||||||
Result<Map<String, List<KCTaskOverviewVO>>> getTaskOverview(Long connectClusterId, String connectorName);
|
|
||||||
Result<List<Properties>> getMM2Configs(Long connectClusterId, String connectorName);
|
|
||||||
|
|
||||||
Result<List<ConnectConfigInfosVO>> validateConnectors(MirrorMakerCreateDTO dto);
|
|
||||||
}
|
|
||||||
@@ -1,653 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.connect.mm2.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.connect.connector.ConnectorManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.connect.mm2.MirrorMakerManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterMirrorMakersOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector.ConnectorCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.mm2.MirrorMakerCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.mm2.MetricsMirrorMakersDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectWorker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.WorkerConnector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.config.ConnectConfigInfos;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector.KSConnectorInfo;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.mm2.MirrorMakerMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.connect.ConnectorPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.mm2.ClusterMirrorMakerOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.mm2.MirrorMakerBaseStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.mm2.MirrorMakerStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin.ConnectConfigInfosVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricLineVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.connect.task.KCTaskOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.*;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.connect.KafkaConnectConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.MirrorMakerUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.OpConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.mm2.MirrorMakerMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.plugin.PluginService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.utils.ApiCallThreadPoolService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static org.apache.kafka.connect.runtime.AbstractStatus.State.RUNNING;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.constant.connect.KafkaConnectConstant.*;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/12/26
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class MirrorMakerManagerImpl implements MirrorMakerManager {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(MirrorMakerManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorService connectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private OpConnectorService opConnectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private WorkerConnectorService workerConnectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private WorkerService workerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorManager connectorManager;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MirrorMakerMetricService mirrorMakerMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectClusterService connectClusterService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PluginService pluginService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> createMirrorMaker(MirrorMakerCreateDTO dto, String operator) {
|
|
||||||
// 检查基本参数
|
|
||||||
Result<Void> rv = this.checkCreateMirrorMakerParamAndUnifyData(dto);
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 创建MirrorSourceConnector
|
|
||||||
Result<Void> sourceConnectResult = connectorManager.createConnector(
|
|
||||||
dto,
|
|
||||||
dto.getCheckpointConnectorConfigs() != null? MirrorMakerUtil.genCheckpointName(dto.getConnectorName()): "",
|
|
||||||
dto.getHeartbeatConnectorConfigs() != null? MirrorMakerUtil.genHeartbeatName(dto.getConnectorName()): "",
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
if (sourceConnectResult.failed()) {
|
|
||||||
// 创建失败, 直接返回
|
|
||||||
return Result.buildFromIgnoreData(sourceConnectResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 创建 checkpoint 任务
|
|
||||||
Result<Void> checkpointResult = Result.buildSuc();
|
|
||||||
if (dto.getCheckpointConnectorConfigs() != null) {
|
|
||||||
checkpointResult = connectorManager.createConnector(
|
|
||||||
new ConnectorCreateDTO(dto.getConnectClusterId(), MirrorMakerUtil.genCheckpointName(dto.getConnectorName()), dto.getCheckpointConnectorConfigs()),
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 创建 heartbeat 任务
|
|
||||||
Result<Void> heartbeatResult = Result.buildSuc();
|
|
||||||
if (dto.getHeartbeatConnectorConfigs() != null) {
|
|
||||||
heartbeatResult = connectorManager.createConnector(
|
|
||||||
new ConnectorCreateDTO(dto.getConnectClusterId(), MirrorMakerUtil.genHeartbeatName(dto.getConnectorName()), dto.getHeartbeatConnectorConfigs()),
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 全都成功
|
|
||||||
if (checkpointResult.successful() && checkpointResult.successful()) {
|
|
||||||
return Result.buildSuc();
|
|
||||||
} else if (checkpointResult.failed() && checkpointResult.failed()) {
|
|
||||||
return Result.buildFromRSAndMsg(
|
|
||||||
ResultStatus.KAFKA_CONNECTOR_OPERATE_FAILED,
|
|
||||||
String.format("创建 checkpoint & heartbeat 失败.%n失败信息分别为：%s%n%n%s", checkpointResult.getMessage(), heartbeatResult.getMessage())
|
|
||||||
);
|
|
||||||
} else if (checkpointResult.failed()) {
|
|
||||||
return Result.buildFromRSAndMsg(
|
|
||||||
ResultStatus.KAFKA_CONNECTOR_OPERATE_FAILED,
|
|
||||||
String.format("创建 checkpoint 失败.%n失败信息分别为：%s", checkpointResult.getMessage())
|
|
||||||
);
|
|
||||||
} else{
|
|
||||||
return Result.buildFromRSAndMsg(
|
|
||||||
ResultStatus.KAFKA_CONNECTOR_OPERATE_FAILED,
|
|
||||||
String.format("创建 heartbeat 失败.%n失败信息分别为：%s", heartbeatResult.getMessage())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> deleteMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, sourceConnectorName);
|
|
||||||
if (connectorPO == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getConnectorNotExist(connectClusterId, sourceConnectorName));
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<Void> rv = Result.buildSuc();
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getCheckpointConnectorName())) {
|
|
||||||
rv = opConnectorService.deleteConnector(connectClusterId, connectorPO.getCheckpointConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getHeartbeatConnectorName())) {
|
|
||||||
rv = opConnectorService.deleteConnector(connectClusterId, connectorPO.getHeartbeatConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
return opConnectorService.deleteConnector(connectClusterId, sourceConnectorName, operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> modifyMirrorMakerConfig(MirrorMakerCreateDTO dto, String operator) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(dto.getConnectClusterId(), dto.getConnectorName());
|
|
||||||
if (connectorPO == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getConnectorNotExist(dto.getConnectClusterId(), dto.getConnectorName()));
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<Void> rv = Result.buildSuc();
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getCheckpointConnectorName()) && dto.getCheckpointConnectorConfigs() != null) {
|
|
||||||
rv = opConnectorService.updateConnectorConfig(dto.getConnectClusterId(), connectorPO.getCheckpointConnectorName(), dto.getCheckpointConnectorConfigs(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getHeartbeatConnectorName()) && dto.getHeartbeatConnectorConfigs() != null) {
|
|
||||||
rv = opConnectorService.updateConnectorConfig(dto.getConnectClusterId(), connectorPO.getHeartbeatConnectorName(), dto.getHeartbeatConnectorConfigs(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
return opConnectorService.updateConnectorConfig(dto.getConnectClusterId(), dto.getConnectorName(), dto.getSuitableConfig(), operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> restartMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, sourceConnectorName);
|
|
||||||
if (connectorPO == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getConnectorNotExist(connectClusterId, sourceConnectorName));
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<Void> rv = Result.buildSuc();
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getCheckpointConnectorName())) {
|
|
||||||
rv = opConnectorService.restartConnector(connectClusterId, connectorPO.getCheckpointConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getHeartbeatConnectorName())) {
|
|
||||||
rv = opConnectorService.restartConnector(connectClusterId, connectorPO.getHeartbeatConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
return opConnectorService.restartConnector(connectClusterId, sourceConnectorName, operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> stopMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, sourceConnectorName);
|
|
||||||
if (connectorPO == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getConnectorNotExist(connectClusterId, sourceConnectorName));
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<Void> rv = Result.buildSuc();
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getCheckpointConnectorName())) {
|
|
||||||
rv = opConnectorService.stopConnector(connectClusterId, connectorPO.getCheckpointConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getHeartbeatConnectorName())) {
|
|
||||||
rv = opConnectorService.stopConnector(connectClusterId, connectorPO.getHeartbeatConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
return opConnectorService.stopConnector(connectClusterId, sourceConnectorName, operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> resumeMirrorMaker(Long connectClusterId, String sourceConnectorName, String operator) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, sourceConnectorName);
|
|
||||||
if (connectorPO == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getConnectorNotExist(connectClusterId, sourceConnectorName));
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<Void> rv = Result.buildSuc();
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getCheckpointConnectorName())) {
|
|
||||||
rv = opConnectorService.resumeConnector(connectClusterId, connectorPO.getCheckpointConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getHeartbeatConnectorName())) {
|
|
||||||
rv = opConnectorService.resumeConnector(connectClusterId, connectorPO.getHeartbeatConnectorName(), operator);
|
|
||||||
}
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
return opConnectorService.resumeConnector(connectClusterId, sourceConnectorName, operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<MirrorMakerStateVO> getMirrorMakerStateVO(Long clusterPhyId) {
|
|
||||||
List<ConnectorPO> connectorPOList = connectorService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
List<WorkerConnector> workerConnectorList = workerConnectorService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
List<ConnectWorker> workerList = workerService.listByKafkaClusterIdFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
return Result.buildSuc(convert2MirrorMakerStateVO(connectorPOList, workerConnectorList, workerList));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<ClusterMirrorMakerOverviewVO> getClusterMirrorMakersOverview(Long clusterPhyId, ClusterMirrorMakersOverviewDTO dto) {
|
|
||||||
List<ConnectorPO> mirrorMakerList = connectorService.listByKafkaClusterIdFromDB(clusterPhyId).stream().filter(elem -> elem.getConnectorClassName().equals(MIRROR_MAKER_SOURCE_CONNECTOR_TYPE)).collect(Collectors.toList());
|
|
||||||
List<ConnectCluster> connectClusterList = connectClusterService.listByKafkaCluster(clusterPhyId);
|
|
||||||
|
|
||||||
|
|
||||||
Result<List<MirrorMakerMetrics>> latestMetricsResult = mirrorMakerMetricService.getLatestMetricsFromES(clusterPhyId,
|
|
||||||
mirrorMakerList.stream().map(elem -> new Tuple<>(elem.getConnectClusterId(), elem.getConnectorName())).collect(Collectors.toList()),
|
|
||||||
dto.getLatestMetricNames());
|
|
||||||
|
|
||||||
if (latestMetricsResult.failed()) {
|
|
||||||
LOGGER.error("method=getClusterMirrorMakersOverview||clusterPhyId={}||result={}||errMsg=get latest metric failed", clusterPhyId, latestMetricsResult);
|
|
||||||
return PaginationResult.buildFailure(latestMetricsResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<ClusterMirrorMakerOverviewVO> mirrorMakerOverviewVOList = this.convert2ClusterMirrorMakerOverviewVO(mirrorMakerList, connectClusterList, latestMetricsResult.getData());
|
|
||||||
|
|
||||||
List<ClusterMirrorMakerOverviewVO> mirrorMakerVOList = this.completeClusterInfo(mirrorMakerOverviewVOList);
|
|
||||||
|
|
||||||
PaginationResult<ClusterMirrorMakerOverviewVO> voPaginationResult = this.pagingMirrorMakerInLocal(mirrorMakerVOList, dto);
|
|
||||||
|
|
||||||
if (voPaginationResult.failed()) {
|
|
||||||
LOGGER.error("method=ClusterMirrorMakerOverviewVO||clusterPhyId={}||result={}||errMsg=pagination in local failed", clusterPhyId, voPaginationResult);
|
|
||||||
|
|
||||||
return PaginationResult.buildFailure(voPaginationResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 查询历史指标
|
|
||||||
Result<List<MetricMultiLinesVO>> lineMetricsResult = mirrorMakerMetricService.listMirrorMakerClusterMetricsFromES(
|
|
||||||
clusterPhyId,
|
|
||||||
this.buildMetricsConnectorsDTO(
|
|
||||||
voPaginationResult.getData().getBizData().stream().map(elem -> new ClusterConnectorDTO(elem.getConnectClusterId(), elem.getConnectorName())).collect(Collectors.toList()),
|
|
||||||
dto.getMetricLines()
|
|
||||||
));
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(
|
|
||||||
this.supplyData2ClusterMirrorMakerOverviewVOList(
|
|
||||||
voPaginationResult.getData().getBizData(),
|
|
||||||
lineMetricsResult.getData()
|
|
||||||
),
|
|
||||||
voPaginationResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<MirrorMakerBaseStateVO> getMirrorMakerState(Long connectClusterId, String connectName) {
|
|
||||||
//mm2任务
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, connectName);
|
|
||||||
if (connectorPO == null){
|
|
||||||
return Result.buildFrom(ResultStatus.NOT_EXIST);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<WorkerConnector> workerConnectorList = workerConnectorService.listFromDB(connectClusterId).stream()
|
|
||||||
.filter(workerConnector -> workerConnector.getConnectorName().equals(connectorPO.getConnectorName())
|
|
||||||
|| (!StringUtils.isBlank(connectorPO.getCheckpointConnectorName()) && workerConnector.getConnectorName().equals(connectorPO.getCheckpointConnectorName()))
|
|
||||||
|| (!StringUtils.isBlank(connectorPO.getHeartbeatConnectorName()) && workerConnector.getConnectorName().equals(connectorPO.getHeartbeatConnectorName())))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
|
|
||||||
MirrorMakerBaseStateVO mirrorMakerBaseStateVO = new MirrorMakerBaseStateVO();
|
|
||||||
mirrorMakerBaseStateVO.setTotalTaskCount(workerConnectorList.size());
|
|
||||||
mirrorMakerBaseStateVO.setAliveTaskCount(workerConnectorList.stream().filter(elem -> elem.getState().equals(RUNNING.name())).collect(Collectors.toList()).size());
|
|
||||||
mirrorMakerBaseStateVO.setWorkerCount(workerConnectorList.stream().collect(Collectors.groupingBy(WorkerConnector::getWorkerId)).size());
|
|
||||||
return Result.buildSuc(mirrorMakerBaseStateVO);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Map<String, List<KCTaskOverviewVO>>> getTaskOverview(Long connectClusterId, String connectorName) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, connectorName);
|
|
||||||
if (connectorPO == null){
|
|
||||||
return Result.buildFrom(ResultStatus.NOT_EXIST);
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<String, List<KCTaskOverviewVO>> listMap = new HashMap<>();
|
|
||||||
List<WorkerConnector> workerConnectorList = workerConnectorService.listFromDB(connectClusterId);
|
|
||||||
if (workerConnectorList.isEmpty()){
|
|
||||||
return Result.buildSuc(listMap);
|
|
||||||
}
|
|
||||||
workerConnectorList.forEach(workerConnector -> {
|
|
||||||
if (workerConnector.getConnectorName().equals(connectorPO.getConnectorName())){
|
|
||||||
listMap.putIfAbsent(KafkaConnectConstant.MIRROR_MAKER_SOURCE_CONNECTOR_TYPE, new ArrayList<>());
|
|
||||||
listMap.get(MIRROR_MAKER_SOURCE_CONNECTOR_TYPE).add(ConvertUtil.obj2Obj(workerConnector, KCTaskOverviewVO.class));
|
|
||||||
} else if (workerConnector.getConnectorName().equals(connectorPO.getCheckpointConnectorName())) {
|
|
||||||
listMap.putIfAbsent(KafkaConnectConstant.MIRROR_MAKER_HEARTBEAT_CONNECTOR_TYPE, new ArrayList<>());
|
|
||||||
listMap.get(MIRROR_MAKER_HEARTBEAT_CONNECTOR_TYPE).add(ConvertUtil.obj2Obj(workerConnector, KCTaskOverviewVO.class));
|
|
||||||
} else if (workerConnector.getConnectorName().equals(connectorPO.getHeartbeatConnectorName())) {
|
|
||||||
listMap.putIfAbsent(KafkaConnectConstant.MIRROR_MAKER_CHECKPOINT_CONNECTOR_TYPE, new ArrayList<>());
|
|
||||||
listMap.get(MIRROR_MAKER_CHECKPOINT_CONNECTOR_TYPE).add(ConvertUtil.obj2Obj(workerConnector, KCTaskOverviewVO.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
return Result.buildSuc(listMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<Properties>> getMM2Configs(Long connectClusterId, String connectorName) {
|
|
||||||
ConnectorPO connectorPO = connectorService.getConnectorFromDB(connectClusterId, connectorName);
|
|
||||||
if (connectorPO == null){
|
|
||||||
return Result.buildFrom(ResultStatus.NOT_EXIST);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<Properties> propList = new ArrayList<>();
|
|
||||||
|
|
||||||
// source
|
|
||||||
Result<KSConnectorInfo> connectorResult = connectorService.getConnectorInfoFromCluster(connectClusterId, connectorPO.getConnectorName());
|
|
||||||
if (connectorResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(connectorResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
Properties props = new Properties();
|
|
||||||
props.putAll(connectorResult.getData().getConfig());
|
|
||||||
propList.add(props);
|
|
||||||
|
|
||||||
// checkpoint
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getCheckpointConnectorName())) {
|
|
||||||
connectorResult = connectorService.getConnectorInfoFromCluster(connectClusterId, connectorPO.getCheckpointConnectorName());
|
|
||||||
if (connectorResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(connectorResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
props = new Properties();
|
|
||||||
props.putAll(connectorResult.getData().getConfig());
|
|
||||||
propList.add(props);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// heartbeat
|
|
||||||
if (!ValidateUtils.isBlank(connectorPO.getHeartbeatConnectorName())) {
|
|
||||||
connectorResult = connectorService.getConnectorInfoFromCluster(connectClusterId, connectorPO.getHeartbeatConnectorName());
|
|
||||||
if (connectorResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(connectorResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
props = new Properties();
|
|
||||||
props.putAll(connectorResult.getData().getConfig());
|
|
||||||
propList.add(props);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(propList);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<ConnectConfigInfosVO>> validateConnectors(MirrorMakerCreateDTO dto) {
|
|
||||||
List<ConnectConfigInfosVO> voList = new ArrayList<>();
|
|
||||||
|
|
||||||
Result<ConnectConfigInfos> infoResult = pluginService.validateConfig(dto.getConnectClusterId(), dto.getSuitableConfig());
|
|
||||||
if (infoResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(infoResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
voList.add(ConvertUtil.obj2Obj(infoResult.getData(), ConnectConfigInfosVO.class));
|
|
||||||
|
|
||||||
if (dto.getHeartbeatConnectorConfigs() != null) {
|
|
||||||
infoResult = pluginService.validateConfig(dto.getConnectClusterId(), dto.getHeartbeatConnectorConfigs());
|
|
||||||
if (infoResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(infoResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
voList.add(ConvertUtil.obj2Obj(infoResult.getData(), ConnectConfigInfosVO.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dto.getCheckpointConnectorConfigs() != null) {
|
|
||||||
infoResult = pluginService.validateConfig(dto.getConnectClusterId(), dto.getCheckpointConnectorConfigs());
|
|
||||||
if (infoResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(infoResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
voList.add(ConvertUtil.obj2Obj(infoResult.getData(), ConnectConfigInfosVO.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(voList);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private MetricsMirrorMakersDTO buildMetricsConnectorsDTO(List<ClusterConnectorDTO> connectorDTOList, MetricDTO metricDTO) {
|
|
||||||
MetricsMirrorMakersDTO dto = ConvertUtil.obj2Obj(metricDTO, MetricsMirrorMakersDTO.class);
|
|
||||||
dto.setConnectorNameList(connectorDTOList == null? new ArrayList<>(): connectorDTOList);
|
|
||||||
|
|
||||||
return dto;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Result<Void> checkCreateMirrorMakerParamAndUnifyData(MirrorMakerCreateDTO dto) {
|
|
||||||
ClusterPhy sourceClusterPhy = clusterPhyService.getClusterByCluster(dto.getSourceKafkaClusterId());
|
|
||||||
if (sourceClusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getSourceKafkaClusterId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
ConnectCluster connectCluster = connectClusterService.getById(dto.getConnectClusterId());
|
|
||||||
if (connectCluster == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getConnectClusterNotExist(dto.getConnectClusterId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
ClusterPhy targetClusterPhy = clusterPhyService.getClusterByCluster(connectCluster.getKafkaClusterPhyId());
|
|
||||||
if (targetClusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getClusterPhyNotExist(connectCluster.getKafkaClusterPhyId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!dto.getSuitableConfig().containsKey(CONNECTOR_CLASS_FILED_NAME)) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "SourceConnector缺少connector.class");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!MIRROR_MAKER_SOURCE_CONNECTOR_TYPE.equals(dto.getSuitableConfig().getProperty(CONNECTOR_CLASS_FILED_NAME))) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "SourceConnector的connector.class类型错误");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dto.getCheckpointConnectorConfigs() != null) {
|
|
||||||
if (!dto.getCheckpointConnectorConfigs().containsKey(CONNECTOR_CLASS_FILED_NAME)) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "CheckpointConnector缺少connector.class");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!MIRROR_MAKER_CHECKPOINT_CONNECTOR_TYPE.equals(dto.getCheckpointConnectorConfigs().getProperty(CONNECTOR_CLASS_FILED_NAME))) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "Checkpoint的connector.class类型错误");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dto.getHeartbeatConnectorConfigs() != null) {
|
|
||||||
if (!dto.getHeartbeatConnectorConfigs().containsKey(CONNECTOR_CLASS_FILED_NAME)) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "HeartbeatConnector缺少connector.class");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!MIRROR_MAKER_HEARTBEAT_CONNECTOR_TYPE.equals(dto.getHeartbeatConnectorConfigs().getProperty(CONNECTOR_CLASS_FILED_NAME))) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "Heartbeat的connector.class类型错误");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
dto.unifyData(
|
|
||||||
sourceClusterPhy.getId(), sourceClusterPhy.getBootstrapServers(), ConvertUtil.str2ObjByJson(sourceClusterPhy.getClientProperties(), Properties.class),
|
|
||||||
targetClusterPhy.getId(), targetClusterPhy.getBootstrapServers(), ConvertUtil.str2ObjByJson(targetClusterPhy.getClientProperties(), Properties.class)
|
|
||||||
);
|
|
||||||
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
private MirrorMakerStateVO convert2MirrorMakerStateVO(List<ConnectorPO> connectorPOList,List<WorkerConnector> workerConnectorList,List<ConnectWorker> workerList){
|
|
||||||
MirrorMakerStateVO mirrorMakerStateVO = new MirrorMakerStateVO();
|
|
||||||
|
|
||||||
List<ConnectorPO> sourceSet = connectorPOList.stream().filter(elem -> elem.getConnectorClassName().equals(MIRROR_MAKER_SOURCE_CONNECTOR_TYPE)).collect(Collectors.toList());
|
|
||||||
mirrorMakerStateVO.setMirrorMakerCount(sourceSet.size());
|
|
||||||
|
|
||||||
Set<Long> connectClusterIdSet = sourceSet.stream().map(ConnectorPO::getConnectClusterId).collect(Collectors.toSet());
|
|
||||||
mirrorMakerStateVO.setWorkerCount(workerList.stream().filter(elem -> connectClusterIdSet.contains(elem.getConnectClusterId())).collect(Collectors.toList()).size());
|
|
||||||
|
|
||||||
List<ConnectorPO> mirrorMakerConnectorList = new ArrayList<>();
|
|
||||||
mirrorMakerConnectorList.addAll(sourceSet);
|
|
||||||
mirrorMakerConnectorList.addAll(connectorPOList.stream().filter(elem -> elem.getConnectorClassName().equals(MIRROR_MAKER_CHECKPOINT_CONNECTOR_TYPE)).collect(Collectors.toList()));
|
|
||||||
mirrorMakerConnectorList.addAll(connectorPOList.stream().filter(elem -> elem.getConnectorClassName().equals(MIRROR_MAKER_HEARTBEAT_CONNECTOR_TYPE)).collect(Collectors.toList()));
|
|
||||||
mirrorMakerStateVO.setTotalConnectorCount(mirrorMakerConnectorList.size());
|
|
||||||
mirrorMakerStateVO.setAliveConnectorCount(mirrorMakerConnectorList.stream().filter(elem -> elem.getState().equals(RUNNING.name())).collect(Collectors.toList()).size());
|
|
||||||
|
|
||||||
Set<String> connectorNameSet = mirrorMakerConnectorList.stream().map(elem -> elem.getConnectorName()).collect(Collectors.toSet());
|
|
||||||
List<WorkerConnector> taskList = workerConnectorList.stream().filter(elem -> connectorNameSet.contains(elem.getConnectorName())).collect(Collectors.toList());
|
|
||||||
mirrorMakerStateVO.setTotalTaskCount(taskList.size());
|
|
||||||
mirrorMakerStateVO.setAliveTaskCount(taskList.stream().filter(elem -> elem.getState().equals(RUNNING.name())).collect(Collectors.toList()).size());
|
|
||||||
|
|
||||||
return mirrorMakerStateVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<ClusterMirrorMakerOverviewVO> convert2ClusterMirrorMakerOverviewVO(List<ConnectorPO> mirrorMakerList, List<ConnectCluster> connectClusterList, List<MirrorMakerMetrics> latestMetric) {
|
|
||||||
List<ClusterMirrorMakerOverviewVO> clusterMirrorMakerOverviewVOList = new ArrayList<>();
|
|
||||||
Map<String, MirrorMakerMetrics> metricsMap = latestMetric.stream().collect(Collectors.toMap(elem -> elem.getConnectClusterId() + "@" + elem.getConnectorName(), Function.identity()));
|
|
||||||
Map<Long, ConnectCluster> connectClusterMap = connectClusterList.stream().collect(Collectors.toMap(elem -> elem.getId(), Function.identity()));
|
|
||||||
|
|
||||||
for (ConnectorPO mirrorMaker : mirrorMakerList) {
|
|
||||||
ClusterMirrorMakerOverviewVO clusterMirrorMakerOverviewVO = new ClusterMirrorMakerOverviewVO();
|
|
||||||
clusterMirrorMakerOverviewVO.setConnectClusterId(mirrorMaker.getConnectClusterId());
|
|
||||||
clusterMirrorMakerOverviewVO.setConnectClusterName(connectClusterMap.get(mirrorMaker.getConnectClusterId()).getName());
|
|
||||||
clusterMirrorMakerOverviewVO.setConnectorName(mirrorMaker.getConnectorName());
|
|
||||||
clusterMirrorMakerOverviewVO.setState(mirrorMaker.getState());
|
|
||||||
clusterMirrorMakerOverviewVO.setCheckpointConnector(mirrorMaker.getCheckpointConnectorName());
|
|
||||||
clusterMirrorMakerOverviewVO.setTaskCount(mirrorMaker.getTaskCount());
|
|
||||||
clusterMirrorMakerOverviewVO.setHeartbeatConnector(mirrorMaker.getHeartbeatConnectorName());
|
|
||||||
clusterMirrorMakerOverviewVO.setLatestMetrics(metricsMap.getOrDefault(mirrorMaker.getConnectClusterId() + "@" + mirrorMaker.getConnectorName(), new MirrorMakerMetrics(mirrorMaker.getConnectClusterId(), mirrorMaker.getConnectorName())));
|
|
||||||
clusterMirrorMakerOverviewVOList.add(clusterMirrorMakerOverviewVO);
|
|
||||||
}
|
|
||||||
return clusterMirrorMakerOverviewVOList;
|
|
||||||
}
|
|
||||||
|
|
||||||
PaginationResult<ClusterMirrorMakerOverviewVO> pagingMirrorMakerInLocal(List<ClusterMirrorMakerOverviewVO> mirrorMakerOverviewVOList, ClusterMirrorMakersOverviewDTO dto) {
|
|
||||||
List<ClusterMirrorMakerOverviewVO> mirrorMakerVOList = PaginationUtil.pageByFuzzyFilter(mirrorMakerOverviewVOList, dto.getSearchKeywords(), Arrays.asList("connectorName"));
|
|
||||||
|
|
||||||
//排序
|
|
||||||
if (!dto.getLatestMetricNames().isEmpty()) {
|
|
||||||
PaginationMetricsUtil.sortMetrics(mirrorMakerVOList, "latestMetrics", dto.getSortMetricNameList(), "connectorName", dto.getSortType());
|
|
||||||
} else {
|
|
||||||
PaginationUtil.pageBySort(mirrorMakerVOList, dto.getSortField(), dto.getSortType(), "connectorName", dto.getSortType());
|
|
||||||
}
|
|
||||||
|
|
||||||
//分页
|
|
||||||
return PaginationUtil.pageBySubData(mirrorMakerVOList, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<ClusterMirrorMakerOverviewVO> supplyData2ClusterMirrorMakerOverviewVOList(List<ClusterMirrorMakerOverviewVO> voList,
|
|
||||||
List<MetricMultiLinesVO> metricLineVOList) {
|
|
||||||
Map<String, List<MetricLineVO>> metricLineMap = new HashMap<>();
|
|
||||||
if (metricLineVOList != null) {
|
|
||||||
for (MetricMultiLinesVO metricMultiLinesVO : metricLineVOList) {
|
|
||||||
metricMultiLinesVO.getMetricLines()
|
|
||||||
.forEach(metricLineVO -> {
|
|
||||||
String key = metricLineVO.getName();
|
|
||||||
List<MetricLineVO> metricLineVOS = metricLineMap.getOrDefault(key, new ArrayList<>());
|
|
||||||
metricLineVOS.add(metricLineVO);
|
|
||||||
metricLineMap.put(key, metricLineVOS);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
voList.forEach(elem -> elem.setMetricLines(metricLineMap.get(elem.getConnectClusterId() + "#" + elem.getConnectorName())));
|
|
||||||
|
|
||||||
return voList;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<ClusterMirrorMakerOverviewVO> completeClusterInfo(List<ClusterMirrorMakerOverviewVO> mirrorMakerVOList) {
|
|
||||||
|
|
||||||
Map<String, KSConnectorInfo> connectorInfoMap = new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
for (ClusterMirrorMakerOverviewVO mirrorMakerVO : mirrorMakerVOList) {
|
|
||||||
ApiCallThreadPoolService.runnableTask(String.format("method=completeClusterInfo||connectClusterId=%d||connectorName=%s||getMirrorMakerInfo", mirrorMakerVO.getConnectClusterId(), mirrorMakerVO.getConnectorName()),
|
|
||||||
3000
|
|
||||||
, () -> {
|
|
||||||
Result<KSConnectorInfo> connectorInfoRet = connectorService.getConnectorInfoFromCluster(mirrorMakerVO.getConnectClusterId(), mirrorMakerVO.getConnectorName());
|
|
||||||
if (connectorInfoRet.hasData()) {
|
|
||||||
connectorInfoMap.put(mirrorMakerVO.getConnectClusterId() + mirrorMakerVO.getConnectorName(), connectorInfoRet.getData());
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
ApiCallThreadPoolService.waitResult();
|
|
||||||
|
|
||||||
List<ClusterMirrorMakerOverviewVO> newMirrorMakerVOList = new ArrayList<>();
|
|
||||||
for (ClusterMirrorMakerOverviewVO mirrorMakerVO : mirrorMakerVOList) {
|
|
||||||
KSConnectorInfo connectorInfo = connectorInfoMap.get(mirrorMakerVO.getConnectClusterId() + mirrorMakerVO.getConnectorName());
|
|
||||||
if (connectorInfo == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
String sourceClusterAlias = connectorInfo.getConfig().get(MIRROR_MAKER_SOURCE_CLUSTER_ALIAS_FIELD_NAME);
|
|
||||||
String targetClusterAlias = connectorInfo.getConfig().get(MIRROR_MAKER_TARGET_CLUSTER_ALIAS_FIELD_NAME);
|
|
||||||
//先默认设置为集群别名
|
|
||||||
mirrorMakerVO.setSourceKafkaClusterName(sourceClusterAlias);
|
|
||||||
mirrorMakerVO.setDestKafkaClusterName(targetClusterAlias);
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(sourceClusterAlias) && CommonUtils.isNumeric(sourceClusterAlias)) {
|
|
||||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(Long.valueOf(sourceClusterAlias));
|
|
||||||
if (clusterPhy != null) {
|
|
||||||
mirrorMakerVO.setSourceKafkaClusterId(clusterPhy.getId());
|
|
||||||
mirrorMakerVO.setSourceKafkaClusterName(clusterPhy.getName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ValidateUtils.isBlank(targetClusterAlias) && CommonUtils.isNumeric(targetClusterAlias)) {
|
|
||||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(Long.valueOf(targetClusterAlias));
|
|
||||||
if (clusterPhy != null) {
|
|
||||||
mirrorMakerVO.setDestKafkaClusterId(clusterPhy.getId());
|
|
||||||
mirrorMakerVO.setDestKafkaClusterName(clusterPhy.getName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newMirrorMakerVOList.add(mirrorMakerVO);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return newMirrorMakerVOList;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.group;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterGroupSummaryDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetDeleteDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetResetDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicPartitionKS;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicConsumedDetailVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
public interface GroupManager {
|
|
||||||
PaginationResult<GroupTopicOverviewVO> pagingGroupMembers(Long clusterPhyId,
|
|
||||||
String topicName,
|
|
||||||
String groupName,
|
|
||||||
String searchTopicKeyword,
|
|
||||||
String searchGroupKeyword,
|
|
||||||
PaginationBaseDTO dto);
|
|
||||||
|
|
||||||
PaginationResult<GroupTopicOverviewVO> pagingGroupTopicMembers(Long clusterPhyId, String groupName, PaginationBaseDTO dto) throws Exception;
|
|
||||||
|
|
||||||
PaginationResult<GroupOverviewVO> pagingClusterGroupsOverview(Long clusterPhyId, ClusterGroupSummaryDTO dto);
|
|
||||||
|
|
||||||
PaginationResult<GroupTopicConsumedDetailVO> pagingGroupTopicConsumedMetrics(Long clusterPhyId,
|
|
||||||
String topicName,
|
|
||||||
String groupName,
|
|
||||||
List<String> latestMetricNames,
|
|
||||||
PaginationSortDTO dto)throws NotExistException, AdminOperateException;
|
|
||||||
|
|
||||||
Result<Set<TopicPartitionKS>> listClusterPhyGroupPartitions(Long clusterPhyId, String groupName, Long startTime, Long endTime);
|
|
||||||
|
|
||||||
Result<Void> resetGroupOffsets(GroupOffsetResetDTO dto, String operator) throws Exception;
|
|
||||||
|
|
||||||
Result<Void> deleteGroupOffsets(GroupOffsetDeleteDTO dto, String operator) throws Exception;
|
|
||||||
|
|
||||||
@Deprecated
|
|
||||||
List<GroupTopicOverviewVO> getGroupTopicOverviewVOList(Long clusterPhyId, List<GroupMemberPO> groupMemberPOList);
|
|
||||||
List<GroupTopicOverviewVO> getGroupTopicOverviewVOList(Long clusterPhyId, List<GroupMemberPO> groupMemberPOList, Integer timeoutUnitMs);
|
|
||||||
}
|
|
||||||
@@ -1,505 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.group.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.group.GroupManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterGroupSummaryDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetDeleteDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetResetDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.partition.PartitionOffsetDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.Group;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopicMember;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSGroupDescription;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSMemberConsumerAssignment;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSMemberDescription;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.offset.KSOffsetSpec;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.group.DeleteGroupParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.group.DeleteGroupTopicParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.group.DeleteGroupTopicPartitionParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicPartitionKS;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicConsumedDetailVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.PaginationConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.GroupConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.OffsetTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.SortTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.group.DeleteGroupTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationMetricsUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.config.KSConfigUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.group.OpGroupService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.GroupMetricVersionItems;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.utils.ApiCallThreadPoolService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.GroupMetricESDAO;
|
|
||||||
import org.apache.kafka.common.ConsumerGroupState;
|
|
||||||
import org.apache.kafka.common.TopicPartition;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.group.GroupTypeEnum.CONNECT_CLUSTER_PROTOCOL_TYPE;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class GroupManagerImpl implements GroupManager {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(GroupManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupService groupService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private OpGroupService opGroupService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PartitionService partitionService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupMetricService groupMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupMetricESDAO groupMetricESDAO;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KSConfigUtils ksConfigUtils;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<GroupTopicOverviewVO> pagingGroupMembers(Long clusterPhyId,
|
|
||||||
String topicName,
|
|
||||||
String groupName,
|
|
||||||
String searchTopicKeyword,
|
|
||||||
String searchGroupKeyword,
|
|
||||||
PaginationBaseDTO dto) {
|
|
||||||
long startTimeUnitMs = System.currentTimeMillis();
|
|
||||||
|
|
||||||
PaginationResult<GroupMemberPO> paginationResult = groupService.pagingGroupMembers(clusterPhyId, topicName, groupName, searchTopicKeyword, searchGroupKeyword, dto);
|
|
||||||
|
|
||||||
if (!paginationResult.hasData()) {
|
|
||||||
return PaginationResult.buildSuc(new ArrayList<>(), paginationResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<GroupTopicOverviewVO> groupTopicVOList = this.getGroupTopicOverviewVOList(
|
|
||||||
clusterPhyId,
|
|
||||||
paginationResult.getData().getBizData(),
|
|
||||||
ksConfigUtils.getApiCallLeftTimeUnitMs(System.currentTimeMillis() - startTimeUnitMs) // 超时时间
|
|
||||||
);
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(groupTopicVOList, paginationResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<GroupTopicOverviewVO> pagingGroupTopicMembers(Long clusterPhyId, String groupName, PaginationBaseDTO dto) throws Exception {
|
|
||||||
long startTimeUnitMs = System.currentTimeMillis();
|
|
||||||
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return PaginationResult.buildFailure(MsgConstant.getClusterPhyNotExist(clusterPhyId), dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
Group group = groupService.getGroupFromKafka(clusterPhy, groupName);
|
|
||||||
|
|
||||||
//没有topicMember则直接返回
|
|
||||||
if (group == null || ValidateUtils.isEmptyList(group.getTopicMembers())) {
|
|
||||||
return PaginationResult.buildSuc(dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
//排序
|
|
||||||
List<GroupTopicMember> groupTopicMembers = PaginationUtil.pageBySort(group.getTopicMembers(), PaginationConstant.DEFAULT_GROUP_TOPIC_SORTED_FIELD, SortTypeEnum.DESC.getSortType());
|
|
||||||
|
|
||||||
//分页
|
|
||||||
PaginationResult<GroupTopicMember> paginationResult = PaginationUtil.pageBySubData(groupTopicMembers, dto);
|
|
||||||
|
|
||||||
List<GroupMemberPO> groupMemberPOList = paginationResult.getData().getBizData().stream().map(elem -> new GroupMemberPO(clusterPhyId, elem.getTopicName(), groupName, group.getState().getState(), elem.getMemberCount())).collect(Collectors.toList());
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(
|
|
||||||
this.getGroupTopicOverviewVOList(
|
|
||||||
clusterPhyId,
|
|
||||||
groupMemberPOList,
|
|
||||||
ksConfigUtils.getApiCallLeftTimeUnitMs(System.currentTimeMillis() - startTimeUnitMs) // 超时时间
|
|
||||||
),
|
|
||||||
paginationResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<GroupOverviewVO> pagingClusterGroupsOverview(Long clusterPhyId, ClusterGroupSummaryDTO dto) {
|
|
||||||
List<Group> groupList = groupService.listClusterGroups(clusterPhyId);
|
|
||||||
|
|
||||||
// 类型转化
|
|
||||||
List<GroupOverviewVO> voList = groupList.stream().map(GroupConverter::convert2GroupOverviewVO).collect(Collectors.toList());
|
|
||||||
|
|
||||||
// 搜索groupName
|
|
||||||
voList = PaginationUtil.pageByFuzzyFilter(voList, dto.getSearchGroupName(), Arrays.asList("name"));
|
|
||||||
|
|
||||||
//搜索topic
|
|
||||||
if (!ValidateUtils.isBlank(dto.getSearchTopicName())) {
|
|
||||||
voList = voList.stream().filter(elem -> {
|
|
||||||
for (String topicName : elem.getTopicNameList()) {
|
|
||||||
if (topicName.contains(dto.getSearchTopicName())) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}).collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 分页 后 返回
|
|
||||||
return PaginationUtil.pageBySubData(voList, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<GroupTopicConsumedDetailVO> pagingGroupTopicConsumedMetrics(Long clusterPhyId,
|
|
||||||
String topicName,
|
|
||||||
String groupName,
|
|
||||||
List<String> latestMetricNames,
|
|
||||||
PaginationSortDTO dto) throws NotExistException, AdminOperateException {
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return PaginationResult.buildFailure(MsgConstant.getClusterPhyNotExist(clusterPhyId), dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 获取消费组消费的TopicPartition列表
|
|
||||||
Map<TopicPartition, Long> consumedOffsetMap = groupService.getGroupOffsetFromKafka(clusterPhyId, groupName);
|
|
||||||
List<Integer> partitionList = consumedOffsetMap.keySet()
|
|
||||||
.stream()
|
|
||||||
.filter(elem -> elem.topic().equals(topicName))
|
|
||||||
.map(elem -> elem.partition())
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
Collections.sort(partitionList);
|
|
||||||
|
|
||||||
// 获取消费组当前运行信息
|
|
||||||
KSGroupDescription groupDescription = groupService.getGroupDescriptionFromKafka(clusterPhy, groupName);
|
|
||||||
|
|
||||||
// 转换存储格式
|
|
||||||
Map<TopicPartition, KSMemberDescription> tpMemberMap = new HashMap<>();
|
|
||||||
|
|
||||||
// 如果不是connect集群
|
|
||||||
if (!groupDescription.protocolType().equals(CONNECT_CLUSTER_PROTOCOL_TYPE)) {
|
|
||||||
for (KSMemberDescription description : groupDescription.members()) {
|
|
||||||
// 如果是 Consumer 的 Description ，则 Assignment 的类型为 KSMemberConsumerAssignment 的
|
|
||||||
KSMemberConsumerAssignment assignment = (KSMemberConsumerAssignment) description.assignment();
|
|
||||||
for (TopicPartition tp : assignment.topicPartitions()) {
|
|
||||||
tpMemberMap.put(tp, description);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 获取指标
|
|
||||||
PaginationResult<GroupMetrics> metricsResult = this.pagingGroupTopicPartitionMetrics(clusterPhyId, groupName, topicName, partitionList, latestMetricNames, dto);
|
|
||||||
if (metricsResult.failed()) {
|
|
||||||
return PaginationResult.buildFailure(metricsResult, dto);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 数据组装
|
|
||||||
List<GroupTopicConsumedDetailVO> voList = new ArrayList<>();
|
|
||||||
for (GroupMetrics groupMetrics: metricsResult.getData().getBizData()) {
|
|
||||||
GroupTopicConsumedDetailVO vo = new GroupTopicConsumedDetailVO();
|
|
||||||
vo.setTopicName(topicName);
|
|
||||||
vo.setPartitionId(groupMetrics.getPartitionId());
|
|
||||||
|
|
||||||
KSMemberDescription ksMemberDescription = tpMemberMap.get(new TopicPartition(topicName, groupMetrics.getPartitionId()));
|
|
||||||
if (ksMemberDescription != null) {
|
|
||||||
vo.setMemberId(ksMemberDescription.consumerId());
|
|
||||||
vo.setHost(ksMemberDescription.host());
|
|
||||||
vo.setClientId(ksMemberDescription.clientId());
|
|
||||||
}
|
|
||||||
|
|
||||||
vo.setLatestMetrics(groupMetrics);
|
|
||||||
voList.add(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(voList, metricsResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Set<TopicPartitionKS>> listClusterPhyGroupPartitions(Long clusterPhyId, String groupName, Long startTime, Long endTime) {
|
|
||||||
try {
|
|
||||||
return Result.buildSuc(groupMetricESDAO.listGroupTopicPartitions(clusterPhyId, groupName, startTime, endTime));
|
|
||||||
}catch (Exception e){
|
|
||||||
return Result.buildFailure(e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> resetGroupOffsets(GroupOffsetResetDTO dto, String operator) throws Exception {
|
|
||||||
Result<Void> rv = this.checkFieldLegal(dto);
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterId());
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getClusterId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
KSGroupDescription description = groupService.getGroupDescriptionFromKafka(clusterPhy, dto.getGroupName());
|
|
||||||
if (ConsumerGroupState.DEAD.equals(description.state()) && !dto.isCreateIfNotExist()) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, "group不存在, 重置失败");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ConsumerGroupState.EMPTY.equals(description.state()) && !ConsumerGroupState.DEAD.equals(description.state())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, String.format("group处于%s, 重置失败(仅Empty | Dead 情况可重置)", GroupStateEnum.getByRawState(description.state()).getState()));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 获取offset
|
|
||||||
Result<Map<TopicPartition, Long>> offsetMapResult = this.getPartitionOffset(dto);
|
|
||||||
if (offsetMapResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(offsetMapResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 重置offset
|
|
||||||
return groupService.resetGroupOffsets(dto.getClusterId(), dto.getGroupName(), offsetMapResult.getData(), operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> deleteGroupOffsets(GroupOffsetDeleteDTO dto, String operator) throws Exception {
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterPhyId());
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getClusterPhyId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// 按照group纬度进行删除
|
|
||||||
if (ValidateUtils.isBlank(dto.getGroupName())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "groupName不允许为空");
|
|
||||||
}
|
|
||||||
if (DeleteGroupTypeEnum.GROUP.getCode().equals(dto.getDeleteType())) {
|
|
||||||
return opGroupService.deleteGroupOffset(
|
|
||||||
new DeleteGroupParam(dto.getClusterPhyId(), dto.getGroupName(), DeleteGroupTypeEnum.GROUP),
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// 按照topic纬度进行删除
|
|
||||||
if (ValidateUtils.isBlank(dto.getTopicName())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "topicName不允许为空");
|
|
||||||
}
|
|
||||||
if (DeleteGroupTypeEnum.GROUP_TOPIC.getCode().equals(dto.getDeleteType())) {
|
|
||||||
return opGroupService.deleteGroupTopicOffset(
|
|
||||||
new DeleteGroupTopicParam(dto.getClusterPhyId(), dto.getGroupName(), DeleteGroupTypeEnum.GROUP, dto.getTopicName()),
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// 按照partition纬度进行删除
|
|
||||||
if (ValidateUtils.isNullOrLessThanZero(dto.getPartitionId())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "partitionId不允许为空或小于0");
|
|
||||||
}
|
|
||||||
if (DeleteGroupTypeEnum.GROUP_TOPIC_PARTITION.getCode().equals(dto.getDeleteType())) {
|
|
||||||
return opGroupService.deleteGroupTopicPartitionOffset(
|
|
||||||
new DeleteGroupTopicPartitionParam(dto.getClusterPhyId(), dto.getGroupName(), DeleteGroupTypeEnum.GROUP, dto.getTopicName(), dto.getPartitionId()),
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "deleteType类型错误");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<GroupTopicOverviewVO> getGroupTopicOverviewVOList(Long clusterPhyId, List<GroupMemberPO> groupMemberPOList) {
|
|
||||||
// 获取指标
|
|
||||||
Result<List<GroupMetrics>> metricsListResult = groupMetricService.listLatestMetricsAggByGroupTopicFromES(
|
|
||||||
clusterPhyId,
|
|
||||||
groupMemberPOList.stream().map(elem -> new GroupTopic(elem.getGroupName(), elem.getTopicName())).collect(Collectors.toList()),
|
|
||||||
Arrays.asList(GroupMetricVersionItems.GROUP_METRIC_LAG),
|
|
||||||
AggTypeEnum.MAX
|
|
||||||
);
|
|
||||||
if (metricsListResult.failed()) {
|
|
||||||
// 如果查询失败，则输出错误信息，但是依旧进行已有数据的返回
|
|
||||||
LOGGER.error("method=completeMetricData||clusterPhyId={}||result={}||errMsg=search es failed", clusterPhyId, metricsListResult);
|
|
||||||
}
|
|
||||||
return this.convert2GroupTopicOverviewVOList(groupMemberPOList, metricsListResult.getData());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<GroupTopicOverviewVO> getGroupTopicOverviewVOList(Long clusterPhyId, List<GroupMemberPO> poList, Integer timeoutUnitMs) {
|
|
||||||
Set<String> requestedGroupSet = new HashSet<>();
|
|
||||||
|
|
||||||
// 获取指标
|
|
||||||
Map<String, Map<String, Float>> groupTopicLagMap = new ConcurrentHashMap<>();
|
|
||||||
poList.forEach(elem -> {
|
|
||||||
if (requestedGroupSet.contains(elem.getGroupName())) {
|
|
||||||
// 该Group已经处理过
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
requestedGroupSet.add(elem.getGroupName());
|
|
||||||
ApiCallThreadPoolService.runnableTask(
|
|
||||||
String.format("clusterPhyId=%d||groupName=%s||msg=getGroupTopicLag", clusterPhyId, elem.getGroupName()),
|
|
||||||
timeoutUnitMs,
|
|
||||||
() -> {
|
|
||||||
Result<List<GroupMetrics>> listResult = groupMetricService.collectGroupMetricsFromKafka(clusterPhyId, elem.getGroupName(), GroupMetricVersionItems.GROUP_METRIC_LAG);
|
|
||||||
if (listResult == null || !listResult.hasData()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<String, Float> lagMetricMap = new HashMap<>();
|
|
||||||
listResult.getData().forEach(item -> {
|
|
||||||
Float newLag = item.getMetric(GroupMetricVersionItems.GROUP_METRIC_LAG);
|
|
||||||
if (newLag == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Float oldLag = lagMetricMap.getOrDefault(item.getTopic(), newLag);
|
|
||||||
lagMetricMap.put(item.getTopic(), Math.max(oldLag, newLag));
|
|
||||||
});
|
|
||||||
|
|
||||||
groupTopicLagMap.put(elem.getGroupName(), lagMetricMap);
|
|
||||||
}
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
ApiCallThreadPoolService.waitResult();
|
|
||||||
|
|
||||||
return this.convert2GroupTopicOverviewVOList(poList, groupTopicLagMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
|
|
||||||
private Result<Void> checkFieldLegal(GroupOffsetResetDTO dto) {
|
|
||||||
if (dto == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "参数为空");
|
|
||||||
}
|
|
||||||
|
|
||||||
Topic topic = topicService.getTopic(dto.getClusterId(), dto.getTopicName());
|
|
||||||
if (topic == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(dto.getClusterId(), dto.getTopicName()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (OffsetTypeEnum.PRECISE_OFFSET.getResetType() == dto.getResetType()
|
|
||||||
&& ValidateUtils.isEmptyList(dto.getOffsetList())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "参数错误，指定offset重置需传offset信息");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (OffsetTypeEnum.PRECISE_TIMESTAMP.getResetType() == dto.getResetType()
|
|
||||||
&& ValidateUtils.isNull(dto.getTimestamp())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "参数错误，指定时间重置需传时间信息");
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Result<Map<TopicPartition, Long>> getPartitionOffset(GroupOffsetResetDTO dto) {
|
|
||||||
if (OffsetTypeEnum.PRECISE_OFFSET.getResetType() == dto.getResetType()) {
|
|
||||||
return Result.buildSuc(dto.getOffsetList().stream().collect(Collectors.toMap(
|
|
||||||
elem -> new TopicPartition(dto.getTopicName(), elem.getPartitionId()),
|
|
||||||
PartitionOffsetDTO::getOffset,
|
|
||||||
(key1 , key2) -> key2
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
KSOffsetSpec offsetSpec = null;
|
|
||||||
if (OffsetTypeEnum.PRECISE_TIMESTAMP.getResetType() == dto.getResetType()) {
|
|
||||||
offsetSpec = KSOffsetSpec.forTimestamp(dto.getTimestamp());
|
|
||||||
} else if (OffsetTypeEnum.EARLIEST.getResetType() == dto.getResetType()) {
|
|
||||||
offsetSpec = KSOffsetSpec.earliest();
|
|
||||||
} else {
|
|
||||||
offsetSpec = KSOffsetSpec.latest();
|
|
||||||
}
|
|
||||||
|
|
||||||
return partitionService.getPartitionOffsetFromKafka(dto.getClusterId(), dto.getTopicName(), offsetSpec);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<GroupTopicOverviewVO> convert2GroupTopicOverviewVOList(List<GroupMemberPO> poList, List<GroupMetrics> metricsList) {
|
|
||||||
if (metricsList == null) {
|
|
||||||
metricsList = new ArrayList<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
// <GroupName, <TopicName, lag>>
|
|
||||||
Map<String, Map<String, Float>> metricsMap = new HashMap<>();
|
|
||||||
metricsList.stream().forEach(elem -> {
|
|
||||||
Float metricValue = elem.getMetrics().get(GroupMetricVersionItems.GROUP_METRIC_LAG);
|
|
||||||
if (metricValue == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
metricsMap.putIfAbsent(elem.getGroup(), new HashMap<>());
|
|
||||||
metricsMap.get(elem.getGroup()).put(elem.getTopic(), metricValue);
|
|
||||||
});
|
|
||||||
|
|
||||||
return this.convert2GroupTopicOverviewVOList(poList, metricsMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<GroupTopicOverviewVO> convert2GroupTopicOverviewVOList(List<GroupMemberPO> poList, Map<String, Map<String, Float>> metricsMap) {
|
|
||||||
List<GroupTopicOverviewVO> voList = new ArrayList<>();
|
|
||||||
for (GroupMemberPO po: poList) {
|
|
||||||
GroupTopicOverviewVO vo = ConvertUtil.obj2Obj(po, GroupTopicOverviewVO.class);
|
|
||||||
if (vo == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Float metricValue = metricsMap.getOrDefault(po.getGroupName(), new HashMap<>()).get(po.getTopicName());
|
|
||||||
if (metricValue != null) {
|
|
||||||
vo.setMaxLag(ConvertUtil.Float2Long(metricValue));
|
|
||||||
}
|
|
||||||
|
|
||||||
voList.add(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
return voList;
|
|
||||||
}
|
|
||||||
|
|
||||||
private PaginationResult<GroupMetrics> pagingGroupTopicPartitionMetrics(Long clusterPhyId,
|
|
||||||
String groupName,
|
|
||||||
String topicName,
|
|
||||||
List<Integer> partitionIdList,
|
|
||||||
List<String> latestMetricNames,
|
|
||||||
PaginationSortDTO dto) {
|
|
||||||
|
|
||||||
|
|
||||||
// 获取Group指标信息
|
|
||||||
Result<List<GroupMetrics>> groupMetricsResult = groupMetricService.collectGroupMetricsFromKafka(clusterPhyId, groupName, latestMetricNames == null ? Arrays.asList() : latestMetricNames);
|
|
||||||
|
|
||||||
|
|
||||||
// 转换Group指标
|
|
||||||
List<GroupMetrics> esGroupMetricsList = groupMetricsResult.hasData() ? groupMetricsResult.getData().stream().filter(elem -> topicName.equals(elem.getTopic())).collect(Collectors.toList()) : new ArrayList<>();
|
|
||||||
Map<Integer, GroupMetrics> esMetricsMap = new HashMap<>();
|
|
||||||
for (GroupMetrics groupMetrics: esGroupMetricsList) {
|
|
||||||
esMetricsMap.put(groupMetrics.getPartitionId(), groupMetrics);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<GroupMetrics> allPartitionGroupMetrics = new ArrayList<>();
|
|
||||||
for (Integer partitionId: partitionIdList) {
|
|
||||||
allPartitionGroupMetrics.add(esMetricsMap.getOrDefault(partitionId, new GroupMetrics(clusterPhyId, groupName, topicName, partitionId)));
|
|
||||||
}
|
|
||||||
|
|
||||||
return PaginationUtil.pageBySubData(
|
|
||||||
(List<GroupMetrics>)PaginationMetricsUtil.sortMetrics(allPartitionGroupMetrics, dto.getSortField(), "partitionId", dto.getSortType()),
|
|
||||||
dto
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.kafkaacl;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.acl.AclAtomDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
public interface KafkaAclManager {
|
|
||||||
Result<Void> batchCreateKafkaAcl(List<AclAtomDTO> dtoList, String operator);
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.kafkaacl.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.kafkaacl.KafkaAclManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.acl.AclAtomDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.KafkaAclConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.acl.OpKafkaAclService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class KafkaAclManagerImpl implements KafkaAclManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(KafkaAclManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private OpKafkaAclService opKafkaAclService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> batchCreateKafkaAcl(List<AclAtomDTO> dtoList, String operator) {
|
|
||||||
log.debug("method=batchCreateKafkaAcl||dtoList={}||operator={}", ConvertUtil.obj2Json(dtoList), operator);
|
|
||||||
|
|
||||||
for (AclAtomDTO dto: dtoList) {
|
|
||||||
Result<Void> rv = opKafkaAclService.createKafkaAcl(KafkaAclConverter.convert2ACLAtomParam(dto), operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.kafkauser;
|
|
||||||
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.kafkauser.ClusterKafkaUserTokenDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.kafkauser.KafkaUserTokenVO;
|
|
||||||
|
|
||||||
public interface KafkaUserManager {
|
|
||||||
/**
|
|
||||||
* 新增KafkaUser
|
|
||||||
*/
|
|
||||||
Result<Void> createKafkaUserWithTokenEncrypted(ClusterKafkaUserTokenDTO dto, String operator);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 修改KafkaUser
|
|
||||||
*/
|
|
||||||
Result<Void> modifyKafkaUserWithTokenEncrypted(ClusterKafkaUserTokenDTO dto, String operator);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查看密码
|
|
||||||
*/
|
|
||||||
Result<KafkaUserTokenVO> getKafkaUserTokenWithEncrypt(Long clusterPhyId, String kafkaUser);
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.kafkauser.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.didiglobal.logi.security.util.PWEncryptUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.kafkauser.KafkaUserManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.kafkauser.ClusterKafkaUserTokenDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkauser.KafkaUser;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.kafkauser.KafkaUserReplaceParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaUserPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.kafkauser.KafkaUserTokenVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.KafkaUserVOConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterAuthTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.AESUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.kafkauser.KafkaUserService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class KafkaUserManagerImpl implements KafkaUserManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(KafkaUserManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KafkaUserService kafkaUserService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> createKafkaUserWithTokenEncrypted(ClusterKafkaUserTokenDTO dto, String operator) {
|
|
||||||
if (!ClusterAuthTypeEnum.isScram(dto.getAuthType())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "不支持该认证方式");
|
|
||||||
}
|
|
||||||
|
|
||||||
String rawToken = AESUtils.decrypt(dto.getToken());
|
|
||||||
if (rawToken == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "KafkaUser密钥解密失败");
|
|
||||||
}
|
|
||||||
|
|
||||||
return kafkaUserService.createKafkaUser(new KafkaUserReplaceParam(dto.getClusterId(), dto.getKafkaUser(), rawToken), operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> modifyKafkaUserWithTokenEncrypted(ClusterKafkaUserTokenDTO dto, String operator) {
|
|
||||||
if (!ClusterAuthTypeEnum.isScram(dto.getAuthType())) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "不支持该认证方式");
|
|
||||||
}
|
|
||||||
|
|
||||||
String rawToken = AESUtils.decrypt(dto.getToken());
|
|
||||||
if (rawToken == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "KafkaUser密钥解密失败");
|
|
||||||
}
|
|
||||||
|
|
||||||
return kafkaUserService.modifyKafkaUser(new KafkaUserReplaceParam(dto.getClusterId(), dto.getKafkaUser(), rawToken), operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<KafkaUserTokenVO> getKafkaUserTokenWithEncrypt(Long clusterPhyId, String kafkaUser) {
|
|
||||||
Result<KafkaUserTokenVO> voResult = this.getKafkaUserToken(clusterPhyId, kafkaUser);
|
|
||||||
if (voResult.failed() || ValidateUtils.isNull(voResult.getData().getToken())) {
|
|
||||||
// 获取失败 或 无密钥信息，则直接返回
|
|
||||||
return voResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 对Token进行加密
|
|
||||||
voResult.getData().setToken(AESUtils.encrypt(voResult.getData().getToken()));
|
|
||||||
return voResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private Result<KafkaUserTokenVO> getKafkaUserToken(Long clusterPhyId, String kafkaUser) {
|
|
||||||
Result<KafkaUser> kafkaUserResult = kafkaUserService.getKafkaUserFromKafka(clusterPhyId, kafkaUser);
|
|
||||||
if (kafkaUserResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(kafkaUserResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
KafkaUserPO kafkaUserPO = kafkaUserService.getKafkaUserFromDB(clusterPhyId, kafkaUser);
|
|
||||||
if (kafkaUserPO == null) {
|
|
||||||
// DB中无数据，则直接返回kafka中查询到的数据
|
|
||||||
return Result.buildSuc(KafkaUserVOConverter.convert2KafkaUserTokenVO(kafkaUserResult.getData(), false, null));
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
String rawToken = PWEncryptUtil.decode(kafkaUserPO.getToken());
|
|
||||||
|
|
||||||
if (kafkaUserService.isTokenEqual2CredentialProps(clusterPhyId, kafkaUserResult.getData().getProps(), rawToken)) {
|
|
||||||
// 与DB中数据一致
|
|
||||||
return Result.buildSuc(KafkaUserVOConverter.convert2KafkaUserTokenVO(kafkaUserResult.getData(), true, rawToken));
|
|
||||||
} else {
|
|
||||||
// 与DB中数据不一致
|
|
||||||
return Result.buildSuc(KafkaUserVOConverter.convert2KafkaUserTokenVO(kafkaUserResult.getData(), false, rawToken));
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
// DB中数据不一致，则直接返回kafka中查询到的数据
|
|
||||||
return Result.buildSuc(KafkaUserVOConverter.convert2KafkaUserTokenVO(kafkaUserResult.getData(), false, null));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.reassign;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.change.CreateChangeReplicasPlanDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.move.CreateMoveReplicaPlanDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.ReassignTopicOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.reassign.plan.ReassignPlanVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.reassign.ReassignTopicOverviewVO;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public interface ReassignManager {
|
|
||||||
/**
|
|
||||||
* 创建迁移计划Json
|
|
||||||
* @param dtoList
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<ReassignPlanVO> createReassignmentPlanJson(List<CreateMoveReplicaPlanDTO> dtoList);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建副本变更Json
|
|
||||||
* @param dtoList
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<ReassignPlanVO> createReplicaChangePlanJson(List<CreateChangeReplicasPlanDTO> dtoList);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 迁移Topic的信息
|
|
||||||
* @param dto
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<List<ReassignTopicOverviewVO>> getReassignmentTopicsOverview(ReassignTopicOverviewDTO dto);
|
|
||||||
}
|
|
||||||
@@ -1,165 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.reassign.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.reassign.ReassignManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.change.CreateChangeReplicasPlanDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.move.CreateMoveReplicaPlanDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.ReassignTopicOverviewDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignPlan;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.reassign.plan.ReassignPlanVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.reassign.ReassignTopicOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.reassign.plan.ReassignTopicPlanVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.ReassignVOConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.TopicMetricVersionItems;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class ReassignManagerImpl implements ReassignManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(ReassignManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ReassignService reassignService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicMetricService topicMetricService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<ReassignPlanVO> createReassignmentPlanJson(List<CreateMoveReplicaPlanDTO> dtoList) {
|
|
||||||
if (ValidateUtils.isEmptyList(dtoList)) {
|
|
||||||
return Result.buildSuc(new ReassignPlanVO(new ArrayList<>()));
|
|
||||||
}
|
|
||||||
|
|
||||||
List<ReassignTopicPlanVO> topicPlanList = new ArrayList<>();
|
|
||||||
for (CreateMoveReplicaPlanDTO planDTO: dtoList) {
|
|
||||||
Result<ReassignPlan> planResult = reassignService.generateReassignmentJson(
|
|
||||||
planDTO.getClusterId(),
|
|
||||||
planDTO.getTopicName(),
|
|
||||||
planDTO.getPartitionIdList(),
|
|
||||||
planDTO.getBrokerIdList(),
|
|
||||||
planDTO.getEnableRackAwareness()
|
|
||||||
);
|
|
||||||
if (planResult.failed()) {
|
|
||||||
// 出错则直接返回错误
|
|
||||||
return Result.buildFromIgnoreData(planResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 转换格式
|
|
||||||
topicPlanList.add(ReassignVOConverter.convert2ReassignTopicPlanVO(planResult.getData()));
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(new ReassignPlanVO(topicPlanList));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<ReassignPlanVO> createReplicaChangePlanJson(List<CreateChangeReplicasPlanDTO> dtoList) {
|
|
||||||
if (ValidateUtils.isEmptyList(dtoList)) {
|
|
||||||
return Result.buildSuc(new ReassignPlanVO(new ArrayList<>()));
|
|
||||||
}
|
|
||||||
|
|
||||||
List<ReassignTopicPlanVO> topicPlanList = new ArrayList<>();
|
|
||||||
for (CreateChangeReplicasPlanDTO planDTO: dtoList) {
|
|
||||||
Result<ReassignPlan> planResult = reassignService.generateReplicaChangeReassignmentJson(
|
|
||||||
planDTO.getClusterId(),
|
|
||||||
planDTO.getTopicName(),
|
|
||||||
planDTO.getNewReplicaNum(),
|
|
||||||
planDTO.getBrokerIdList()
|
|
||||||
);
|
|
||||||
if (planResult.failed()) {
|
|
||||||
// 出错则直接返回错误
|
|
||||||
return Result.buildFromIgnoreData(planResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 转换格式
|
|
||||||
topicPlanList.add(ReassignVOConverter.convert2ReassignTopicPlanVO(planResult.getData()));
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(new ReassignPlanVO(topicPlanList));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<ReassignTopicOverviewVO>> getReassignmentTopicsOverview(ReassignTopicOverviewDTO dto) {
|
|
||||||
Map<String, Topic> topicMap = topicService.listTopicsFromDB(dto.getClusterId()).stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity()));
|
|
||||||
|
|
||||||
Map<String, ReassignTopicOverviewVO> voMap = new HashMap<>();
|
|
||||||
for (String topicName: dto.getTopicNameList()) {
|
|
||||||
Topic topic = topicMap.get(topicName);
|
|
||||||
if (topic == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(dto.getClusterId(), topicName));
|
|
||||||
}
|
|
||||||
|
|
||||||
ReassignTopicOverviewVO vo = ConvertUtil.obj2Obj(topic, ReassignTopicOverviewVO.class);
|
|
||||||
vo.setPartitionIdList(new ArrayList<>(topic.getPartitionMap().keySet()));
|
|
||||||
vo.setRetentionMs(topic.getRetentionMs());
|
|
||||||
vo.setLatestDaysAvgBytesInList(new ArrayList<>());
|
|
||||||
vo.setLatestDaysMaxBytesInList(new ArrayList<>());
|
|
||||||
voMap.put(topicName, vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
Long now = System.currentTimeMillis();
|
|
||||||
|
|
||||||
// 补充近三天指标
|
|
||||||
for (int idx = 0; idx < 3; ++idx) {
|
|
||||||
Long startTime = now - (idx + 1) * 24L * 60L * 60L * 1000L;
|
|
||||||
Long endTime = now - idx * 24L * 60L * 60L * 1000L;
|
|
||||||
|
|
||||||
// 查询avg指标
|
|
||||||
Result<Map<String, MetricPointVO>> avgMetricMapResult = topicMetricService.getAggMetricPointFromES(
|
|
||||||
dto.getClusterId(),
|
|
||||||
dto.getTopicNameList(),
|
|
||||||
TopicMetricVersionItems.TOPIC_METRIC_BYTES_IN,
|
|
||||||
AggTypeEnum.AVG,
|
|
||||||
startTime,
|
|
||||||
endTime
|
|
||||||
);
|
|
||||||
Map<String, MetricPointVO> avgMetricMap = avgMetricMapResult.hasData()? avgMetricMapResult.getData(): new HashMap<>();
|
|
||||||
avgMetricMap.values().forEach(elem -> elem.setTimeStamp(endTime));
|
|
||||||
|
|
||||||
// 查询max指标
|
|
||||||
Result<Map<String, MetricPointVO>> maxMetricMapResult = topicMetricService.getAggMetricPointFromES(
|
|
||||||
dto.getClusterId(),
|
|
||||||
dto.getTopicNameList(),
|
|
||||||
TopicMetricVersionItems.TOPIC_METRIC_BYTES_IN,
|
|
||||||
AggTypeEnum.MAX,
|
|
||||||
startTime,
|
|
||||||
endTime
|
|
||||||
);
|
|
||||||
Map<String, MetricPointVO> maxMetricMap = maxMetricMapResult.hasData()? maxMetricMapResult.getData(): new HashMap<>();
|
|
||||||
|
|
||||||
// 补充到vo中
|
|
||||||
this.supplyLatestMetrics(voMap, avgMetricMap, maxMetricMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(new ArrayList<>(voMap.values()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private void supplyLatestMetrics(Map<String, ReassignTopicOverviewVO> voMap,
|
|
||||||
Map<String, MetricPointVO> avgMetricMap,
|
|
||||||
Map<String, MetricPointVO> maxMetricMap) {
|
|
||||||
for (Map.Entry<String, ReassignTopicOverviewVO> entry: voMap.entrySet()) {
|
|
||||||
entry.getValue().getLatestDaysAvgBytesInList().add(avgMetricMap.get(entry.getKey()));
|
|
||||||
entry.getValue().getLatestDaysMaxBytesInList().add(maxMetricMap.get(entry.getKey()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.self;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.self.SelfMetricsVO;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
public interface SelfManager {
|
|
||||||
Result<SelfMetricsVO> metrics();
|
|
||||||
|
|
||||||
Result<Properties> version();
|
|
||||||
}
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.self.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.didiglobal.logi.security.common.vo.user.UserBriefVO;
|
|
||||||
import com.didiglobal.logi.security.service.UserService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.self.SelfManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.self.SelfMetricsVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.GitPropUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.NetUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.km.KmNodeService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class SelfManagerImpl implements SelfManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(SelfManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private UserService userService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KmNodeService kmNodeService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<SelfMetricsVO> metrics() {
|
|
||||||
SelfMetricsVO vo = new SelfMetricsVO();
|
|
||||||
|
|
||||||
// ks自身信息
|
|
||||||
vo.setKsIp(NetUtils.localIp());
|
|
||||||
vo.setKsClusterKey(NetUtils.localMac());
|
|
||||||
|
|
||||||
List<UserBriefVO> userBriefVOList = userService.getAllUserBriefList();
|
|
||||||
vo.setKsUserCount(ValidateUtils.isNull(userBriefVOList)? 0: userBriefVOList.size());
|
|
||||||
vo.setKsServerIps(kmNodeService.listKmHosts());
|
|
||||||
|
|
||||||
// 纳管集群信息
|
|
||||||
vo.setKafkaClusterCount(clusterPhyService.listAllClusters().size());
|
|
||||||
vo.setKafkaBrokerCount(brokerService.countAllBrokers());
|
|
||||||
|
|
||||||
return Result.buildSuc(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Properties> version() {
|
|
||||||
return Result.buildSuc(GitPropUtil.getProps());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.topic;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicExpansionDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
|
|
||||||
public interface OpTopicManager {
|
|
||||||
/**
|
|
||||||
* 创建Topic
|
|
||||||
*/
|
|
||||||
Result<Void> createTopic(TopicCreateDTO dto, String operator);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 删除Topic
|
|
||||||
*/
|
|
||||||
Result<Void> deleteTopicCombineRes(Long clusterPhyId, String topicName, String operator);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 扩分区
|
|
||||||
*/
|
|
||||||
Result<Void> expandTopic(TopicExpansionDTO dto, String operator);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 清空Topic
|
|
||||||
*/
|
|
||||||
Result<Void> truncateTopic(Long clusterPhyId, String topicName, String operator);
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.topic;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaTopicDefaultConfig;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public interface TopicConfigManager {
|
|
||||||
/**
|
|
||||||
* 获取Topic默认配置
|
|
||||||
* @param clusterPhyId 物理集群ID
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<List<KafkaTopicDefaultConfig>> getDefaultTopicConfig(Long clusterPhyId);
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.topic;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicRecordDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicBrokersPartitionsSummaryVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.broker.TopicBrokerAllVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.partition.TopicPartitionVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public interface TopicStateManager {
|
|
||||||
TopicBrokerAllVO getTopicBrokerAll(Long clusterPhyId, String topicName, String searchBrokerHost) throws NotExistException;
|
|
||||||
|
|
||||||
Result<List<TopicRecordVO>> getTopicMessages(Long clusterPhyId, String topicName, TopicRecordDTO dto) throws AdminOperateException;
|
|
||||||
|
|
||||||
Result<TopicStateVO> getTopicState(Long clusterPhyId, String topicName);
|
|
||||||
|
|
||||||
Result<List<TopicPartitionVO>> getTopicPartitions(Long clusterPhyId, String topicName, List<String> metricsNames);
|
|
||||||
|
|
||||||
Result<TopicBrokersPartitionsSummaryVO> getTopicBrokersPartitionsSummary(Long clusterPhyId, String topicName);
|
|
||||||
|
|
||||||
PaginationResult<GroupTopicOverviewVO> pagingTopicGroupsOverview(Long clusterPhyId, String topicName, String searchGroupName, PaginationBaseDTO dto);
|
|
||||||
}
|
|
||||||
@@ -1,266 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.topic.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.topic.OpTopicManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicCreateDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicExpansionDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaTopicConfigParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicCreateParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicPartitionExpandParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicTruncateParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.*;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.kafka.KafkaReplicaAssignUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.OpTopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import kafka.admin.AdminUtils;
|
|
||||||
import kafka.admin.BrokerMetadata;
|
|
||||||
import org.apache.kafka.common.config.TopicConfig;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
|
||||||
import org.springframework.transaction.interceptor.TransactionAspectSupport;
|
|
||||||
import scala.Option;
|
|
||||||
import scala.collection.Seq;
|
|
||||||
import scala.jdk.javaapi.CollectionConverters;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class OpTopicManagerImpl implements OpTopicManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(OpTopicManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private OpTopicService opTopicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PartitionService partitionService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicConfigService topicConfigService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> createTopic(TopicCreateDTO dto, String operator) {
|
|
||||||
log.info("method=createTopic||param={}||operator={}.", dto, operator);
|
|
||||||
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterId());
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getClusterId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 构造assignmentMap
|
|
||||||
scala.collection.Map<Object, Seq<Object>> rawAssignmentMap = AdminUtils.assignReplicasToBrokers(
|
|
||||||
this.buildBrokerMetadataSeq(dto.getClusterId(), dto.getBrokerIdList()),
|
|
||||||
dto.getPartitionNum(),
|
|
||||||
dto.getReplicaNum(),
|
|
||||||
-1,
|
|
||||||
-1
|
|
||||||
);
|
|
||||||
|
|
||||||
// 类型转换
|
|
||||||
Map<Integer, List<Integer>> assignmentMap = new HashMap<>();
|
|
||||||
rawAssignmentMap.
|
|
||||||
toStream().
|
|
||||||
foreach(elem -> assignmentMap.put(
|
|
||||||
(Integer) elem._1,
|
|
||||||
CollectionConverters.asJava(elem._2).stream().map(item -> (Integer)item).collect(Collectors.toList()))
|
|
||||||
);
|
|
||||||
|
|
||||||
// 创建Topic
|
|
||||||
Result<Void> createTopicRes = opTopicService.createTopic(
|
|
||||||
new TopicCreateParam(
|
|
||||||
dto.getClusterId(),
|
|
||||||
dto.getTopicName(),
|
|
||||||
new HashMap<String, String>((Map) dto.getProperties()),
|
|
||||||
assignmentMap,
|
|
||||||
dto.getDescription()
|
|
||||||
),
|
|
||||||
operator
|
|
||||||
);
|
|
||||||
if (createTopicRes.successful()){
|
|
||||||
try{
|
|
||||||
FutureUtil.quickStartupFutureUtil.submitTask(() -> {
|
|
||||||
BackoffUtils.backoff(3000);
|
|
||||||
Result<List<Partition>> partitionsResult = partitionService.listPartitionsFromKafka(clusterPhy, dto.getTopicName());
|
|
||||||
if (partitionsResult.successful()){
|
|
||||||
partitionService.updatePartitions(clusterPhy.getId(), dto.getTopicName(), partitionsResult.getData(), new ArrayList<>());
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}catch (Exception e) {
|
|
||||||
log.error("method=createTopic||param={}||operator={}||msg=add partition to db failed||errMsg=exception", dto, operator, e);
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, "Topic创建成功，但记录Partition到DB中失败，等待定时任务同步partition信息");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return createTopicRes;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> deleteTopicCombineRes(Long clusterPhyId, String topicName, String operator) {
|
|
||||||
// 删除Topic
|
|
||||||
Result<Void> rv = opTopicService.deleteTopic(new TopicParam(clusterPhyId, topicName), operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 删除Topic相关的ACL信息
|
|
||||||
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
@Transactional
|
|
||||||
public Result<Void> expandTopic(TopicExpansionDTO dto, String operator) {
|
|
||||||
Topic topic = topicService.getTopic(dto.getClusterId(), dto.getTopicName());
|
|
||||||
if (topic == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(dto.getClusterId(), dto.getTopicName()));
|
|
||||||
}
|
|
||||||
|
|
||||||
TopicPartitionExpandParam expandParam = new TopicPartitionExpandParam(
|
|
||||||
dto.getClusterId(),
|
|
||||||
dto.getTopicName(),
|
|
||||||
topic.getPartitionMap(),
|
|
||||||
this.generateNewPartitionAssignment(dto.getClusterId(), topic, dto.getBrokerIdList(), dto.getIncPartitionNum())
|
|
||||||
);
|
|
||||||
|
|
||||||
// 更新DB分区数信息, 其他信息交由后台任务进行更新
|
|
||||||
Result<Void> rv = topicService.updatePartitionNum(topic.getClusterPhyId(), topic.getTopicName(), topic.getPartitionNum() + dto.getIncPartitionNum());
|
|
||||||
if (rv.failed()){
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
rv = opTopicService.expandTopic(expandParam, operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> truncateTopic(Long clusterPhyId, String topicName, String operator) {
|
|
||||||
// 增加delete配置
|
|
||||||
Result<Tuple<Boolean, String>> rt = this.addDeleteConfigIfNotExist(clusterPhyId, topicName, operator);
|
|
||||||
if (rt.failed()) {
|
|
||||||
log.error("method=truncateTopic||clusterPhyId={}||topicName={}||operator={}||result={}||msg=get config from kafka failed", clusterPhyId, topicName, operator, rt);
|
|
||||||
return Result.buildFromIgnoreData(rt);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 清空Topic
|
|
||||||
Result<Void> rv = opTopicService.truncateTopic(new TopicTruncateParam(clusterPhyId, topicName, KafkaConstant.TOPICK_TRUNCATE_DEFAULT_OFFSET), operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
log.error("method=truncateTopic||clusterPhyId={}||topicName={}||originConfig={}||operator={}||result={}||msg=truncate topic failed", clusterPhyId, topicName, rt.getData().v2(), operator, rv);
|
|
||||||
// config被修改了，则错误提示需要提醒一下，否则直接返回错误
|
|
||||||
return rt.getData().v1() ? Result.buildFailure(rv.getCode(), rv.getMessage() + "\t\n" + String.format("Topic的CleanupPolicy已被修改，需要手动恢复为%s", rt.getData().v2())) : rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 恢复compact配置
|
|
||||||
rv = this.recoverConfigIfChanged(clusterPhyId, topicName, rt.getData().v1(), rt.getData().v2(), operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
log.error("method=truncateTopic||clusterPhyId={}||topicName={}||originConfig={}||operator={}||result={}||msg=truncate topic success but recover config failed", clusterPhyId, topicName, rt.getData().v2(), operator, rv);
|
|
||||||
// config被修改了，则错误提示需要提醒一下，否则直接返回错误
|
|
||||||
return Result.buildFailure(rv.getCode(), String.format("Topic清空操作已成功，但是恢复CleanupPolicy配置失败，需要手动恢复为%s。", rt.getData().v2()) + "\t\n" + rv.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private Result<Tuple<Boolean, String>> addDeleteConfigIfNotExist(Long clusterPhyId, String topicName, String operator) {
|
|
||||||
// 获取Topic配置
|
|
||||||
Result<Map<String, String>> configMapResult = topicConfigService.getTopicConfigFromKafka(clusterPhyId, topicName);
|
|
||||||
if (configMapResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(configMapResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
String cleanupPolicyValue = configMapResult.getData().getOrDefault(TopicConfig.CLEANUP_POLICY_CONFIG, "");
|
|
||||||
List<String> cleanupPolicyValueList = CommonUtils.string2StrList(cleanupPolicyValue);
|
|
||||||
if (cleanupPolicyValueList.size() == 1 && cleanupPolicyValueList.contains(TopicConfig.CLEANUP_POLICY_DELETE)) {
|
|
||||||
// 不需要修改
|
|
||||||
return Result.buildSuc(new Tuple<>(Boolean.FALSE, cleanupPolicyValue));
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<String, String> changedConfigMap = new HashMap<>(1);
|
|
||||||
changedConfigMap.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
|
|
||||||
|
|
||||||
Result<Void> rv = topicConfigService.modifyTopicConfig(new KafkaTopicConfigParam(clusterPhyId, topicName, changedConfigMap), operator);
|
|
||||||
if (rv.failed()) {
|
|
||||||
// 修改失败
|
|
||||||
return Result.buildFromIgnoreData(rv);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(new Tuple<>(Boolean.TRUE, cleanupPolicyValue));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Result<Void> recoverConfigIfChanged(Long clusterPhyId, String topicName, Boolean changed, String originValue, String operator) {
|
|
||||||
if (!changed) {
|
|
||||||
// 没有修改，直接返回
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 恢复配置
|
|
||||||
Map<String, String> changedConfigMap = new HashMap<>(1);
|
|
||||||
changedConfigMap.put(TopicConfig.CLEANUP_POLICY_CONFIG, originValue);
|
|
||||||
|
|
||||||
return topicConfigService.modifyTopicConfig(new KafkaTopicConfigParam(clusterPhyId, topicName, changedConfigMap), operator);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Seq<BrokerMetadata> buildBrokerMetadataSeq(Long clusterPhyId, final List<Integer> selectedBrokerIdList) {
|
|
||||||
// 选取Broker列表
|
|
||||||
List<Broker> brokerList = brokerService.listAliveBrokersFromDB(clusterPhyId).stream().filter( elem ->
|
|
||||||
selectedBrokerIdList == null || selectedBrokerIdList.contains(elem.getBrokerId())
|
|
||||||
).collect(Collectors.toList());
|
|
||||||
|
|
||||||
List<BrokerMetadata> brokerMetadataList = new ArrayList<>();
|
|
||||||
for (Broker broker: brokerList) {
|
|
||||||
brokerMetadataList.add(new BrokerMetadata(broker.getBrokerId(), Option.apply(broker.getRack())));
|
|
||||||
}
|
|
||||||
|
|
||||||
return CollectionConverters.asScala(brokerMetadataList);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<Integer, List<Integer>> generateNewPartitionAssignment(Long clusterPhyId, Topic topic, List<Integer> brokerIdList, Integer incPartitionNum) {
|
|
||||||
if (ValidateUtils.isEmptyList(brokerIdList)) {
|
|
||||||
// 如果brokerId列表为空，则获取当前集群存活的Broker列表
|
|
||||||
brokerIdList = brokerService.listAliveBrokersFromDB(clusterPhyId).stream().map( elem -> elem.getBrokerId()).collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<Integer, String> brokerRackMap = new HashMap<>();
|
|
||||||
for (Broker broker: brokerService.listAliveBrokersFromDB(clusterPhyId)) {
|
|
||||||
if (brokerIdList != null && !brokerIdList.contains(broker.getBrokerId())) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
brokerRackMap.put(broker.getBrokerId(), broker.getRack() == null? "": broker.getRack());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 生成分配规则
|
|
||||||
return KafkaReplicaAssignUtil.generateNewPartitionAssignment(brokerRackMap, topic.getPartitionMap(), incPartitionNum);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.topic.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.topic.TopicConfigManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaConfigDetail;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaTopicDefaultConfig;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.broker.BrokerParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.KafkaConfigConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseKafkaVersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class TopicConfigManagerImpl extends BaseKafkaVersionControlService implements TopicConfigManager {
|
|
||||||
private static final ILog log = LogFactory.getLog(TopicConfigManagerImpl.class);
|
|
||||||
|
|
||||||
private static final String GET_DEFAULT_TOPIC_CONFIG = "getDefaultTopicConfig";
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerConfigService brokerConfigService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicConfigService topicConfigService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected VersionItemTypeEnum getVersionItemType() {
|
|
||||||
return VersionItemTypeEnum.SERVICE_OP_TOPIC_CONFIG;
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
private void init() {
|
|
||||||
registerVCHandler(GET_DEFAULT_TOPIC_CONFIG, V_0_10_0_0, V_0_11_0_0, "getDefaultTopicConfigByLocal", this::getDefaultTopicConfigByLocal);
|
|
||||||
registerVCHandler(GET_DEFAULT_TOPIC_CONFIG, V_0_11_0_0, V_MAX, "getDefaultTopicConfigByClient", this::getDefaultTopicConfigByClient);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<KafkaTopicDefaultConfig>> getDefaultTopicConfig(Long clusterPhyId) {
|
|
||||||
try {
|
|
||||||
List<Broker> aliveBrokerList = brokerService.listAliveBrokersFromDB(clusterPhyId);
|
|
||||||
Integer aliveBrokerId = null;
|
|
||||||
if (!aliveBrokerList.isEmpty()) {
|
|
||||||
aliveBrokerId = aliveBrokerList.get(0).getBrokerId();
|
|
||||||
}
|
|
||||||
|
|
||||||
return (Result<List<KafkaTopicDefaultConfig>>) doVCHandler(clusterPhyId, GET_DEFAULT_TOPIC_CONFIG, new BrokerParam(clusterPhyId, aliveBrokerId));
|
|
||||||
} catch (VCHandlerNotExistException e) {
|
|
||||||
return Result.buildFailure(e.getResultStatus());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Result<List<KafkaTopicDefaultConfig>> getDefaultTopicConfigByLocal(VersionItemParam itemParam) {
|
|
||||||
BrokerParam brokerParam = (BrokerParam) itemParam;
|
|
||||||
return Result.buildSuc(KafkaConfigConverter.convert2KafkaTopicDefaultConfigList(
|
|
||||||
topicConfigService.getConfigNamesAndDocs(brokerParam.getClusterPhyId()),
|
|
||||||
new HashMap<>()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Result<List<KafkaTopicDefaultConfig>> getDefaultTopicConfigByClient(VersionItemParam itemParam) {
|
|
||||||
BrokerParam brokerParam = (BrokerParam) itemParam;
|
|
||||||
|
|
||||||
Result<List<KafkaConfigDetail>> defaultConfigResult = brokerConfigService.getBrokerConfigDetailFromKafka(brokerParam.getClusterPhyId(), brokerParam.getBrokerId());
|
|
||||||
if (defaultConfigResult.failed()) {
|
|
||||||
// 获取配置错误，但是不直接返回
|
|
||||||
log.error("method=getDefaultTopicConfigByClient||param={}||result={}.", brokerParam, defaultConfigResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(KafkaConfigConverter.convert2KafkaTopicDefaultConfigList(
|
|
||||||
topicConfigService.getConfigNamesAndDocs(brokerParam.getClusterPhyId()),
|
|
||||||
!defaultConfigResult.hasData()?
|
|
||||||
new HashMap<>():
|
|
||||||
defaultConfigResult.getData().stream().filter(elem -> !ValidateUtils.isNull(elem.getValue())).collect(Collectors.toMap(KafkaConfigDetail::getName, KafkaConfigDetail::getValue))
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,488 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.topic.impl;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.group.GroupManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.topic.TopicStateManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicRecordDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.offset.KSOffsetSpec;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.broker.BrokerReplicaSummaryVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicBrokersPartitionsSummaryVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicStateVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.broker.TopicBrokerAllVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.broker.TopicBrokerSingleVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.partition.TopicPartitionVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.PaginationConstant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.converter.TopicVOConverter;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.OffsetTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.SortTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.config.KSConfigUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.TopicMetricVersionItems;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.utils.ApiCallThreadPoolService;
|
|
||||||
import org.apache.kafka.clients.consumer.*;
|
|
||||||
import org.apache.kafka.common.TopicPartition;
|
|
||||||
import org.apache.kafka.common.config.TopicConfig;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class TopicStateManagerImpl implements TopicStateManager {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(TopicStateManagerImpl.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PartitionService partitionService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PartitionMetricService partitionMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicMetricService topicMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicConfigService topicConfigService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupService groupService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupManager groupManager;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KSConfigUtils ksConfigUtils;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public TopicBrokerAllVO getTopicBrokerAll(Long clusterPhyId, String topicName, String searchBrokerHost) throws NotExistException {
|
|
||||||
Topic topic = topicService.getTopic(clusterPhyId, topicName);
|
|
||||||
|
|
||||||
List<Partition> partitionList = partitionService.listPartitionByTopic(clusterPhyId, topicName);
|
|
||||||
Map<Integer, List<Partition>> brokerIdPartitionListMap = this.convert2BrokerIdPartitionListMap(partitionList);
|
|
||||||
|
|
||||||
Map<Integer, Broker> brokerMap = brokerService.listAllBrokerByTopic(clusterPhyId, topicName).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
|
|
||||||
|
|
||||||
TopicBrokerAllVO allVO = new TopicBrokerAllVO();
|
|
||||||
|
|
||||||
allVO.setTotal(topic.getBrokerIdSet().size());
|
|
||||||
allVO.setLive((int)brokerMap.values().stream().filter(Broker::alive).count());
|
|
||||||
allVO.setDead(allVO.getTotal() - allVO.getLive());
|
|
||||||
|
|
||||||
allVO.setPartitionCount(topic.getPartitionNum());
|
|
||||||
allVO.setBrokerPartitionStateList(new ArrayList<>());
|
|
||||||
allVO.setUnderReplicatedPartitionIdList(new ArrayList<>());
|
|
||||||
allVO.setNoLeaderPartitionIdList(new ArrayList<>());
|
|
||||||
|
|
||||||
// 补充无Leader及未同步的分区
|
|
||||||
for (Partition partition: partitionList) {
|
|
||||||
if (partition.getLeaderBrokerId() == null || Constant.INVALID_CODE == partition.getLeaderBrokerId()) {
|
|
||||||
allVO.getNoLeaderPartitionIdList().add(partition.getPartitionId());
|
|
||||||
}
|
|
||||||
if (partition.getInSyncReplicaList().size() != partition.getAssignReplicaList().size()) {
|
|
||||||
allVO.getUnderReplicatedPartitionIdList().add(partition.getPartitionId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 补充Broker中分区的详情
|
|
||||||
for (Integer brokerId: topic.getBrokerIdSet()) {
|
|
||||||
Broker broker = brokerMap.get(brokerId);
|
|
||||||
if (!ValidateUtils.isBlank(searchBrokerHost) && (broker == null || !broker.getHost().contains(searchBrokerHost))) {
|
|
||||||
// 不满足搜索的要求，则直接略过该Broker
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
allVO.getBrokerPartitionStateList().add(this.getTopicBrokerSingle(clusterPhyId, topicName, brokerIdPartitionListMap, brokerId, broker));
|
|
||||||
}
|
|
||||||
|
|
||||||
return allVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<TopicRecordVO>> getTopicMessages(Long clusterPhyId, String topicName, TopicRecordDTO dto) throws AdminOperateException {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
// 获取集群
|
|
||||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
|
|
||||||
if (clusterPhy == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterPhyId));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 获取分区beginOffset
|
|
||||||
Result<Map<TopicPartition, Long>> beginOffsetsMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, dto.getFilterPartitionId(), KSOffsetSpec.earliest());
|
|
||||||
if (beginOffsetsMapResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(beginOffsetsMapResult);
|
|
||||||
}
|
|
||||||
// 获取分区endOffset
|
|
||||||
Result<Map<TopicPartition, Long>> endOffsetsMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, dto.getFilterPartitionId(), KSOffsetSpec.latest());
|
|
||||||
if (endOffsetsMapResult.failed()) {
|
|
||||||
return Result.buildFromIgnoreData(endOffsetsMapResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 数据采集
|
|
||||||
List<TopicRecordVO> voList = this.getTopicMessages(clusterPhy, topicName, beginOffsetsMapResult.getData(), endOffsetsMapResult.getData(), startTime, dto);
|
|
||||||
|
|
||||||
// 排序
|
|
||||||
if (ValidateUtils.isBlank(dto.getSortType())) {
|
|
||||||
// 默认按时间倒序排序
|
|
||||||
dto.setSortType(SortTypeEnum.DESC.getSortType());
|
|
||||||
}
|
|
||||||
if (ValidateUtils.isBlank(dto.getSortField())) {
|
|
||||||
// 默认按照timestampUnitMs字段排序
|
|
||||||
dto.setSortField(PaginationConstant.TOPIC_RECORDS_TIME_SORTED_FIELD);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (PaginationConstant.TOPIC_RECORDS_TIME_SORTED_FIELD.equals(dto.getSortField())) {
|
|
||||||
// 如果是时间类型，则第二排序规则是offset
|
|
||||||
PaginationUtil.pageBySort(voList, dto.getSortField(), dto.getSortType(), PaginationConstant.TOPIC_RECORDS_OFFSET_SORTED_FIELD, dto.getSortType());
|
|
||||||
} else {
|
|
||||||
// 如果是非时间类型，则第二排序规则是时间
|
|
||||||
PaginationUtil.pageBySort(voList, dto.getSortField(), dto.getSortType(), PaginationConstant.TOPIC_RECORDS_TIME_SORTED_FIELD, dto.getSortType());
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(voList.subList(0, Math.min(dto.getMaxRecords(), voList.size())));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<TopicStateVO> getTopicState(Long clusterPhyId, String topicName) {
|
|
||||||
Topic topic = topicService.getTopic(clusterPhyId, topicName);
|
|
||||||
if (topic == null) {
|
|
||||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(clusterPhyId, topicName));
|
|
||||||
}
|
|
||||||
|
|
||||||
List<Partition> partitionList = partitionService.listPartitionByTopic(clusterPhyId, topicName);
|
|
||||||
if (partitionList == null) {
|
|
||||||
partitionList = new ArrayList<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
TopicStateVO vo = new TopicStateVO();
|
|
||||||
|
|
||||||
// 分区信息
|
|
||||||
vo.setPartitionCount(topic.getPartitionNum());
|
|
||||||
vo.setAllPartitionHaveLeader(partitionList.stream().filter(elem -> elem.getLeaderBrokerId().equals(-1)).count() <= 0);
|
|
||||||
|
|
||||||
// 副本信息
|
|
||||||
vo.setReplicaFactor(topic.getReplicaNum());
|
|
||||||
vo.setAllReplicaInSync(partitionList.stream().filter(elem -> elem.getInSyncReplicaList().size() != topic.getReplicaNum()).count() <= 0);
|
|
||||||
|
|
||||||
// 配置信息
|
|
||||||
Map<String, String> topicConfigMap = new HashMap<>();
|
|
||||||
Result<Map<String, String>> configResult = topicConfigService.getTopicConfigFromKafka(clusterPhyId, topicName);
|
|
||||||
if (configResult.hasData()) {
|
|
||||||
topicConfigMap = configResult.getData();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 最小副本
|
|
||||||
Integer minIsr = ConvertUtil.string2Integer(topicConfigMap.get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG));
|
|
||||||
if (minIsr == null) {
|
|
||||||
vo.setMinimumIsr(null);
|
|
||||||
vo.setAllPartitionMatchAtMinIsr(null);
|
|
||||||
} else {
|
|
||||||
vo.setMinimumIsr(minIsr);
|
|
||||||
vo.setAllPartitionMatchAtMinIsr(partitionList.stream().filter(elem -> elem.getInSyncReplicaList().size() < minIsr).count() <= 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 压缩方式
|
|
||||||
String cleanupPolicy = topicConfigMap.get(TopicConfig.CLEANUP_POLICY_CONFIG);
|
|
||||||
if (ValidateUtils.isBlank(cleanupPolicy)) {
|
|
||||||
vo.setCompacted(null);
|
|
||||||
} else {
|
|
||||||
vo.setCompacted(cleanupPolicy.contains(TopicConfig.CLEANUP_POLICY_COMPACT));
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<TopicPartitionVO>> getTopicPartitions(Long clusterPhyId, String topicName, List<String> metricsNames) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
List<Partition> partitionList = partitionService.listPartitionByTopic(clusterPhyId, topicName);
|
|
||||||
if (ValidateUtils.isEmptyList(partitionList)) {
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<Integer, PartitionMetrics> metricsMap = new HashMap<>();
|
|
||||||
ApiCallThreadPoolService.runnableTask(
|
|
||||||
String.format("clusterPhyId=%d||topicName=%s||method=getTopicPartitions", clusterPhyId, topicName),
|
|
||||||
ksConfigUtils.getApiCallLeftTimeUnitMs(System.currentTimeMillis() - startTime),
|
|
||||||
() -> {
|
|
||||||
Result<List<PartitionMetrics>> metricsResult = partitionMetricService.collectPartitionsMetricsFromKafka(clusterPhyId, topicName, metricsNames);
|
|
||||||
if (metricsResult.failed()) {
|
|
||||||
// 仅打印错误日志，但是不直接返回错误
|
|
||||||
LOGGER.error(
|
|
||||||
"method=getTopicPartitions||clusterPhyId={}||topicName={}||result={}||msg=get metrics from kafka failed",
|
|
||||||
clusterPhyId, topicName, metricsResult
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (PartitionMetrics metrics: metricsResult.getData()) {
|
|
||||||
metricsMap.put(metrics.getPartitionId(), metrics);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
boolean finished = ApiCallThreadPoolService.waitResultAndReturnFinished(1);
|
|
||||||
|
|
||||||
if (!finished && metricsMap.isEmpty()) {
|
|
||||||
// 未完成 -> 打印日志
|
|
||||||
LOGGER.error("method=getTopicPartitions||clusterPhyId={}||topicName={}||msg=get metrics from kafka failed", clusterPhyId, topicName);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<TopicPartitionVO> voList = new ArrayList<>();
|
|
||||||
for (Partition partition: partitionList) {
|
|
||||||
voList.add(TopicVOConverter.convert2TopicPartitionVO(partition, metricsMap.get(partition.getPartitionId())));
|
|
||||||
}
|
|
||||||
return Result.buildSuc(voList);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<TopicBrokersPartitionsSummaryVO> getTopicBrokersPartitionsSummary(Long clusterPhyId, String topicName) {
|
|
||||||
List<Partition> partitionList = partitionService.listPartitionByTopic(clusterPhyId, topicName);
|
|
||||||
Map<Integer, Broker> brokerMap = brokerService.listAllBrokerByTopic(clusterPhyId, topicName).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
|
|
||||||
|
|
||||||
TopicBrokersPartitionsSummaryVO vo = new TopicBrokersPartitionsSummaryVO();
|
|
||||||
|
|
||||||
// Broker统计信息
|
|
||||||
vo.setBrokerCount(brokerMap.size());
|
|
||||||
vo.setLiveBrokerCount((int)brokerMap.values().stream().filter(Broker::alive).count());
|
|
||||||
vo.setDeadBrokerCount(vo.getBrokerCount() - vo.getLiveBrokerCount());
|
|
||||||
|
|
||||||
// Partition统计信息
|
|
||||||
vo.setPartitionCount(partitionList.size());
|
|
||||||
vo.setNoLeaderPartitionCount(0);
|
|
||||||
vo.setUnderReplicatedPartitionCount(0);
|
|
||||||
|
|
||||||
// 补充无Leader及未同步的分区
|
|
||||||
for (Partition partition: partitionList) {
|
|
||||||
if (partition.getLeaderBrokerId() == null || Constant.INVALID_CODE == partition.getLeaderBrokerId()) {
|
|
||||||
vo.setNoLeaderPartitionCount(vo.getNoLeaderPartitionCount() + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (partition.getInSyncReplicaList().size() != partition.getAssignReplicaList().size()) {
|
|
||||||
vo.setUnderReplicatedPartitionCount(vo.getUnderReplicatedPartitionCount() + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(vo);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public PaginationResult<GroupTopicOverviewVO> pagingTopicGroupsOverview(Long clusterPhyId, String topicName, String searchGroupName, PaginationBaseDTO dto) {
|
|
||||||
long startTimeUnitMs = System.currentTimeMillis();
|
|
||||||
|
|
||||||
PaginationResult<GroupMemberPO> paginationResult = groupService.pagingGroupMembers(clusterPhyId, topicName, "", "", searchGroupName, dto);
|
|
||||||
|
|
||||||
if (!paginationResult.hasData()) {
|
|
||||||
return PaginationResult.buildSuc(new ArrayList<>(), paginationResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<GroupTopicOverviewVO> groupTopicVOList = groupManager.getGroupTopicOverviewVOList(
|
|
||||||
clusterPhyId,
|
|
||||||
paginationResult.getData().getBizData(),
|
|
||||||
ksConfigUtils.getApiCallLeftTimeUnitMs(System.currentTimeMillis() - startTimeUnitMs) // 超时时间
|
|
||||||
);
|
|
||||||
|
|
||||||
return PaginationResult.buildSuc(groupTopicVOList, paginationResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private boolean checkIfIgnore(ConsumerRecord<String, String> consumerRecord, String filterKey, String filterValue) {
|
|
||||||
if (filterKey != null && consumerRecord.key() == null) {
|
|
||||||
// ignore
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (filterKey != null && consumerRecord.key() != null && !consumerRecord.key().contains(filterKey)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (filterValue != null && consumerRecord.value() == null) {
|
|
||||||
// ignore
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (filterValue != null && consumerRecord.value() != null && !consumerRecord.value().contains(filterValue));
|
|
||||||
}
|
|
||||||
|
|
||||||
private TopicBrokerSingleVO getTopicBrokerSingle(Long clusterPhyId,
|
|
||||||
String topicName,
|
|
||||||
Map<Integer, List<Partition>> brokerIdPartitionListMap,
|
|
||||||
Integer brokerId,
|
|
||||||
Broker broker) {
|
|
||||||
TopicBrokerSingleVO singleVO = new TopicBrokerSingleVO();
|
|
||||||
singleVO.setBrokerId(brokerId);
|
|
||||||
singleVO.setHost(broker != null? broker.getHost(): null);
|
|
||||||
singleVO.setAlive(broker != null && broker.alive());
|
|
||||||
|
|
||||||
TopicMetrics metrics = topicMetricService.getTopicLatestMetricsFromES(clusterPhyId, brokerId, topicName, Arrays.asList(
|
|
||||||
TopicMetricVersionItems.TOPIC_METRIC_BYTES_IN,
|
|
||||||
TopicMetricVersionItems.TOPIC_METRIC_BYTES_OUT
|
|
||||||
));
|
|
||||||
if (metrics != null) {
|
|
||||||
singleVO.setBytesInOneMinuteRate(metrics.getMetrics().get(TopicMetricVersionItems.TOPIC_METRIC_BYTES_IN));
|
|
||||||
singleVO.setBytesOutOneMinuteRate(metrics.getMetrics().get(TopicMetricVersionItems.TOPIC_METRIC_BYTES_OUT));
|
|
||||||
}
|
|
||||||
singleVO.setReplicaList(this.getBrokerReplicaSummaries(brokerId, brokerIdPartitionListMap.getOrDefault(brokerId, new ArrayList<>())));
|
|
||||||
return singleVO;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<BrokerReplicaSummaryVO> getBrokerReplicaSummaries(Integer brokerId, List<Partition> partitionList) {
|
|
||||||
List<BrokerReplicaSummaryVO> voList = new ArrayList<>();
|
|
||||||
for (Partition partition: partitionList) {
|
|
||||||
BrokerReplicaSummaryVO summaryVO = new BrokerReplicaSummaryVO();
|
|
||||||
summaryVO.setTopicName(partition.getTopicName());
|
|
||||||
summaryVO.setPartitionId(partition.getPartitionId());
|
|
||||||
summaryVO.setLeaderBrokerId(partition.getLeaderBrokerId());
|
|
||||||
summaryVO.setIsLeaderReplace(brokerId.equals(partition.getLeaderBrokerId()));
|
|
||||||
summaryVO.setInSync(partition.getInSyncReplicaList().contains(brokerId));
|
|
||||||
voList.add(summaryVO);
|
|
||||||
}
|
|
||||||
return voList;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<Integer, List<Partition>> convert2BrokerIdPartitionListMap(List<Partition> partitionList) {
|
|
||||||
Map<Integer, List<Partition>> brokerIdPartitionListMap = new HashMap<>();
|
|
||||||
for (Partition partition: partitionList) {
|
|
||||||
for (Integer brokerId: partition.getAssignReplicaList()) {
|
|
||||||
brokerIdPartitionListMap.putIfAbsent(brokerId, new ArrayList<>());
|
|
||||||
brokerIdPartitionListMap.get(brokerId).add(partition);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return brokerIdPartitionListMap;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Properties generateClientProperties(ClusterPhy clusterPhy, Integer maxPollRecords) {
|
|
||||||
Properties props = ConvertUtil.str2ObjByJson(clusterPhy.getClientProperties(), Properties.class);
|
|
||||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, clusterPhy.getBootstrapServers());
|
|
||||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
|
|
||||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
|
|
||||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
|
|
||||||
|
|
||||||
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Math.max(2, Math.min(5, maxPollRecords)));
|
|
||||||
return props;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<TopicRecordVO> getTopicMessages(ClusterPhy clusterPhy,
|
|
||||||
String topicName,
|
|
||||||
Map<TopicPartition, Long> beginOffsetsMap,
|
|
||||||
Map<TopicPartition, Long> endOffsetsMap,
|
|
||||||
long startTime,
|
|
||||||
TopicRecordDTO dto) throws AdminOperateException {
|
|
||||||
List<TopicRecordVO> voList = new ArrayList<>();
|
|
||||||
|
|
||||||
try (KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(this.generateClientProperties(clusterPhy, dto.getMaxRecords()))) {
|
|
||||||
// 移动到指定位置
|
|
||||||
long maxMessage = this.assignAndSeekToSpecifiedOffset(kafkaConsumer, beginOffsetsMap, endOffsetsMap, dto);
|
|
||||||
|
|
||||||
// 这里需要减去 KafkaConstant.POLL_ONCE_TIMEOUT_UNIT_MS 是因为poll一次需要耗时，如果这里不减去，则可能会导致poll之后，超过要求的时间
|
|
||||||
while (System.currentTimeMillis() - startTime <= dto.getPullTimeoutUnitMs() && voList.size() < maxMessage) {
|
|
||||||
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(KafkaConstant.POLL_ONCE_TIMEOUT_UNIT_MS));
|
|
||||||
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
|
|
||||||
if (this.checkIfIgnore(consumerRecord, dto.getFilterKey(), dto.getFilterValue())) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
voList.add(TopicVOConverter.convert2TopicRecordVO(topicName, consumerRecord));
|
|
||||||
if (voList.size() >= dto.getMaxRecords()) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 超时则返回
|
|
||||||
if (System.currentTimeMillis() - startTime + KafkaConstant.POLL_ONCE_TIMEOUT_UNIT_MS > dto.getPullTimeoutUnitMs()
|
|
||||||
|| voList.size() > dto.getMaxRecords()) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return voList;
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error("method=getTopicMessages||clusterPhyId={}||topicName={}||param={}||errMsg=exception", clusterPhy.getId(), topicName, dto, e);
|
|
||||||
|
|
||||||
throw new AdminOperateException(e.getMessage(), e, ResultStatus.KAFKA_OPERATE_FAILED);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private long assignAndSeekToSpecifiedOffset(KafkaConsumer<String, String> kafkaConsumer,
|
|
||||||
Map<TopicPartition, Long> beginOffsetsMap,
|
|
||||||
Map<TopicPartition, Long> endOffsetsMap,
|
|
||||||
TopicRecordDTO dto) {
|
|
||||||
List<TopicPartition> partitionList = new ArrayList<>();
|
|
||||||
long maxMessage = 0;
|
|
||||||
for (Map.Entry<TopicPartition, Long> entry : endOffsetsMap.entrySet()) {
|
|
||||||
long begin = beginOffsetsMap.get(entry.getKey());
|
|
||||||
long end = entry.getValue();
|
|
||||||
if (begin == end){
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
maxMessage += end - begin;
|
|
||||||
partitionList.add(entry.getKey());
|
|
||||||
}
|
|
||||||
maxMessage = Math.min(maxMessage, dto.getMaxRecords());
|
|
||||||
kafkaConsumer.assign(partitionList);
|
|
||||||
|
|
||||||
Map<TopicPartition, OffsetAndTimestamp> partitionOffsetAndTimestampMap = new HashMap<>();
|
|
||||||
// 获取指定时间每个分区的offset（按指定开始时间查询消息时）
|
|
||||||
if (OffsetTypeEnum.PRECISE_TIMESTAMP.getResetType() == dto.getFilterOffsetReset()) {
|
|
||||||
Map<TopicPartition, Long> timestampsToSearch = new HashMap<>();
|
|
||||||
partitionList.forEach(topicPartition -> timestampsToSearch.put(topicPartition, dto.getStartTimestampUnitMs()));
|
|
||||||
partitionOffsetAndTimestampMap = kafkaConsumer.offsetsForTimes(timestampsToSearch);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (TopicPartition partition : partitionList) {
|
|
||||||
if (OffsetTypeEnum.EARLIEST.getResetType() == dto.getFilterOffsetReset()) {
|
|
||||||
// 重置到最旧
|
|
||||||
kafkaConsumer.seek(partition, beginOffsetsMap.get(partition));
|
|
||||||
} else if (OffsetTypeEnum.PRECISE_TIMESTAMP.getResetType() == dto.getFilterOffsetReset()) {
|
|
||||||
// 重置到指定时间
|
|
||||||
kafkaConsumer.seek(partition, partitionOffsetAndTimestampMap.get(partition).offset());
|
|
||||||
} else if (OffsetTypeEnum.PRECISE_OFFSET.getResetType() == dto.getFilterOffsetReset()) {
|
|
||||||
// 重置到指定位置
|
|
||||||
|
|
||||||
} else {
|
|
||||||
// 默认，重置到最新
|
|
||||||
kafkaConsumer.seek(partition, Math.max(beginOffsetsMap.get(partition), endOffsetsMap.get(partition) - dto.getMaxRecords()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return maxMessage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.version;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.UserMetricConfigDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.config.metric.UserMetricConfigVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.version.VersionItemVO;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
public interface VersionControlManager {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询当前所有的兼容性(指标、前端操作)配置信息
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<Map<String, VersionItemVO>> listAllVersionItem();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取当前ks所有支持的kafka版本
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<Map<String, Long>> listAllKafkaVersions();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取全部集群 clusterId 中类型为 type 的指标，不论支持不支持
|
|
||||||
* @param clusterId
|
|
||||||
* @param type
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<List<VersionItemVO>> listKafkaClusterVersionControlItem(Long clusterId, Integer type);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取当前用户设置的用于展示的指标配置
|
|
||||||
* @param clusterId
|
|
||||||
* @param type
|
|
||||||
* @param operator
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<List<UserMetricConfigVO>> listUserMetricItem(Long clusterId, Integer type, String operator);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 更新用户配置的指标项
|
|
||||||
* @param clusterId
|
|
||||||
* @param type
|
|
||||||
* @param userMetricConfigDTO
|
|
||||||
* @param operator
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
Result<Void> updateUserMetricItem(Long clusterId, Integer type,
|
|
||||||
UserMetricConfigDTO userMetricConfigDTO, String operator);
|
|
||||||
}
|
|
||||||
@@ -1,354 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.biz.version.impl;
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
|
||||||
import com.alibaba.fastjson.TypeReference;
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.didiglobal.logi.security.common.dto.config.ConfigDTO;
|
|
||||||
import com.didiglobal.logi.security.service.ConfigService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.biz.version.VersionControlManager;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDetailDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.UserMetricConfigDTO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.metric.UserMetricConfig;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.config.metric.UserMetricConfigVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.version.VersionItemVO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.VersionUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.util.StringUtils;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.V_MAX;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.BrokerMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.ClusterMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.GroupMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.TopicMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.connect.MirrorMakerMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.connect.ConnectClusterMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.connect.ConnectorMetricVersionItems.*;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.ZookeeperMetricVersionItems.*;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class VersionControlManagerImpl implements VersionControlManager {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(VersionControlManagerImpl.class);
|
|
||||||
|
|
||||||
private static final String NOT_SUPPORT_DESC = "，(该指标只支持%s及以上的版本)";
|
|
||||||
private static final String NOT_SUPPORT_DESC1 = "，(该指标只支持%s及以上和%s以下的版本)";
|
|
||||||
|
|
||||||
private static final String CONFIG_GROUP = "UserMetricConfig";
|
|
||||||
|
|
||||||
Set<UserMetricConfig> defaultMetrics = new HashSet<>();
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
// topic
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_HEALTH_STATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_FAILED_FETCH_REQ, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_FAILED_PRODUCE_REQ, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_UNDER_REPLICA_PARTITIONS, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_TOTAL_PRODUCE_REQUESTS, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_IN, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_OUT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_REJECTED, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_MESSAGE_IN, true));
|
|
||||||
|
|
||||||
// cluster
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_HEALTH_STATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_ACTIVE_CONTROLLER_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_BYTES_IN, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_BYTES_OUT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_CONNECTIONS, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_MESSAGES_IN, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_PARTITIONS_NO_LEADER, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_PARTITION_URP, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_LOG_SIZE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_PRODUCE_REQ, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_REQ_QUEUE_SIZE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_RES_QUEUE_SIZE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_GROUP_REBALANCES, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_JOB_RUNNING, true));
|
|
||||||
|
|
||||||
// group
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_OFFSET_CONSUMED, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_LAG, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_STATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_HEALTH_STATE, true));
|
|
||||||
|
|
||||||
// broker
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_HEALTH_STATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_CONNECTION_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_MESSAGE_IN, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_NETWORK_RPO_AVG_IDLE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_REQ_AVG_IDLE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_PRODUCE_REQ, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_REQ_QUEUE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_RES_QUEUE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_LEADERS_SKEW, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_UNDER_REPLICATE_PARTITION, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_PARTITIONS_SKEW, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_IN, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_OUT, true));
|
|
||||||
|
|
||||||
// zookeeper
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_HEALTH_STATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_HEALTH_CHECK_PASSED, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_HEALTH_CHECK_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_MAX_REQUEST_LATENCY, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_OUTSTANDING_REQUESTS, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_NODE_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_WATCH_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_NUM_ALIVE_CONNECTIONS, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_PACKETS_RECEIVED, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_PACKETS_SENT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_EPHEMERALS_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_APPROXIMATE_DATA_SIZE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_OPEN_FILE_DESCRIPTOR_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_KAFKA_ZK_DISCONNECTS_PER_SEC, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_KAFKA_ZK_SYNC_CONNECTS_PER_SEC, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_ZOOKEEPER.getCode(), ZOOKEEPER_METRIC_KAFKA_ZK_REQUEST_LATENCY_99TH, true));
|
|
||||||
|
|
||||||
// mm2
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_MIRROR_MAKER.getCode(), MIRROR_MAKER_METRIC_BYTE_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_MIRROR_MAKER.getCode(), MIRROR_MAKER_METRIC_BYTE_RATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_MIRROR_MAKER.getCode(), MIRROR_MAKER_METRIC_RECORD_AGE_MS_MAX, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_MIRROR_MAKER.getCode(), MIRROR_MAKER_METRIC_RECORD_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_MIRROR_MAKER.getCode(), MIRROR_MAKER_METRIC_RECORD_RATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_MIRROR_MAKER.getCode(), MIRROR_MAKER_METRIC_REPLICATION_LATENCY_MS_MAX, true));
|
|
||||||
|
|
||||||
// Connect Cluster
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_CONNECTOR_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_TASK_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_CONNECTOR_STARTUP_ATTEMPTS_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_CONNECTOR_STARTUP_FAILURE_PERCENTAGE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_CONNECTOR_STARTUP_FAILURE_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_TASK_STARTUP_ATTEMPTS_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_TASK_STARTUP_FAILURE_PERCENTAGE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_TASK_STARTUP_FAILURE_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CLUSTER.getCode(), CONNECT_CLUSTER_METRIC_COLLECT_COST_TIME, true));
|
|
||||||
|
|
||||||
|
|
||||||
// Connect Connector
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_HEALTH_STATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_HEALTH_CHECK_PASSED, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_HEALTH_CHECK_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_COLLECT_COST_TIME, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_CONNECTOR_TOTAL_TASK_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_CONNECTOR_RUNNING_TASK_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_CONNECTOR_FAILED_TASK_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SOURCE_RECORD_ACTIVE_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SOURCE_RECORD_POLL_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SOURCE_RECORD_WRITE_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SINK_RECORD_ACTIVE_COUNT, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SINK_RECORD_READ_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SINK_RECORD_SEND_TOTAL, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_DEADLETTERQUEUE_PRODUCE_FAILURES, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_DEADLETTERQUEUE_PRODUCE_REQUESTS, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_TOTAL_ERRORS_LOGGED, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SOURCE_RECORD_POLL_RATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SOURCE_RECORD_WRITE_RATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SINK_RECORD_READ_RATE, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_CONNECT_CONNECTOR.getCode(), CONNECTOR_METRIC_SINK_RECORD_SEND_RATE, true));
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConfigService configService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Map<String, VersionItemVO>> listAllVersionItem() {
|
|
||||||
List<VersionItemVO> allVersionItemVO = new ArrayList<>();
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_TOPIC.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CLUSTER.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_GROUP.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_BROKER.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_PARTITION.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_REPLICATION.getCode()), VersionItemVO.class));
|
|
||||||
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_ZOOKEEPER.getCode()), VersionItemVO.class));
|
|
||||||
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CONNECT_CLUSTER.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CONNECT_CONNECTOR.getCode()), VersionItemVO.class));
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CONNECT_MIRROR_MAKER.getCode()), VersionItemVO.class));
|
|
||||||
|
|
||||||
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(WEB_OP.getCode()), VersionItemVO.class));
|
|
||||||
|
|
||||||
Map<String, VersionItemVO> map = allVersionItemVO.stream().collect(
|
|
||||||
Collectors.toMap(
|
|
||||||
u -> u.getType() + "@" + u.getName(),
|
|
||||||
Function.identity(),
|
|
||||||
(v1, v2) -> v1)
|
|
||||||
);
|
|
||||||
|
|
||||||
return Result.buildSuc(map);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Map<String, Long>> listAllKafkaVersions() {
|
|
||||||
return Result.buildSuc(VersionEnum.allVersionsWithOutMax());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<VersionItemVO>> listKafkaClusterVersionControlItem(Long clusterId, Integer type) {
|
|
||||||
List<VersionControlItem> allItem = versionControlService.listVersionControlItem(type);
|
|
||||||
List<VersionItemVO> versionItemVOS = new ArrayList<>();
|
|
||||||
|
|
||||||
String versionStr = clusterPhyService.getVersionFromCacheFirst(clusterId);
|
|
||||||
|
|
||||||
for (VersionControlItem item : allItem){
|
|
||||||
VersionItemVO itemVO = ConvertUtil.obj2Obj(item, VersionItemVO.class);
|
|
||||||
boolean support = versionControlService.isClusterSupport(versionStr, item);
|
|
||||||
|
|
||||||
itemVO.setSupport(support);
|
|
||||||
itemVO.setDesc(itemSupportDesc(item, support));
|
|
||||||
|
|
||||||
versionItemVOS.add(itemVO);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result.buildSuc(versionItemVOS);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<List<UserMetricConfigVO>> listUserMetricItem(Long clusterId, Integer type, String operator) {
|
|
||||||
Result<List<VersionItemVO>> ret = listKafkaClusterVersionControlItem(clusterId, type);
|
|
||||||
if(null == ret || ret.failed()){
|
|
||||||
return Result.buildFail();
|
|
||||||
}
|
|
||||||
|
|
||||||
List<UserMetricConfigVO> userMetricConfigVOS = new ArrayList<>();
|
|
||||||
List<VersionItemVO> allVersionItemVOs = ret.getData();
|
|
||||||
Set<UserMetricConfig> userMetricConfigs = getUserMetricConfig(operator);
|
|
||||||
|
|
||||||
Map<String, UserMetricConfig> userMetricConfigMap = userMetricConfigs.stream().collect(
|
|
||||||
Collectors.toMap(u -> u.getType() + "@" + u.getMetric(), Function.identity() ));
|
|
||||||
|
|
||||||
for(VersionItemVO itemVO : allVersionItemVOs){
|
|
||||||
UserMetricConfigVO userMetricConfigVO = new UserMetricConfigVO();
|
|
||||||
|
|
||||||
int itemType = itemVO.getType();
|
|
||||||
String metric = itemVO.getName();
|
|
||||||
|
|
||||||
UserMetricConfig umc = userMetricConfigMap.get(itemType + "@" + metric);
|
|
||||||
userMetricConfigVO.setSet(null != umc && umc.isSet());
|
|
||||||
if (umc != null) {
|
|
||||||
userMetricConfigVO.setRank(umc.getRank());
|
|
||||||
}
|
|
||||||
userMetricConfigVO.setName(itemVO.getName());
|
|
||||||
userMetricConfigVO.setType(itemVO.getType());
|
|
||||||
userMetricConfigVO.setDesc(itemVO.getDesc());
|
|
||||||
userMetricConfigVO.setMinVersion(itemVO.getMinVersion());
|
|
||||||
userMetricConfigVO.setMaxVersion(itemVO.getMaxVersion());
|
|
||||||
userMetricConfigVO.setSupport(itemVO.getSupport());
|
|
||||||
|
|
||||||
userMetricConfigVOS.add(userMetricConfigVO);
|
|
||||||
}
|
|
||||||
|
|
||||||
LOGGER.debug("method=listUserMetricItem||clusterId={}||type={}||operator={}||userMetricConfigs={}||userMetricConfigVO={}",
|
|
||||||
clusterId, type, operator, JSON.toJSONString(userMetricConfigs), JSON.toJSONString(userMetricConfigVOS));
|
|
||||||
|
|
||||||
return Result.buildSuc(userMetricConfigVOS);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Result<Void> updateUserMetricItem(Long clusterId, Integer type, UserMetricConfigDTO dto, String operator) {
|
|
||||||
Map<String, Boolean> metricsSetMap = dto.getMetricsSet();
|
|
||||||
|
|
||||||
//转换metricDetailDTOList
|
|
||||||
List<MetricDetailDTO> metricDetailDTOList = dto.getMetricDetailDTOList();
|
|
||||||
Map<String, MetricDetailDTO> metricDetailMap = new HashMap<>();
|
|
||||||
if (metricDetailDTOList != null && !metricDetailDTOList.isEmpty()) {
|
|
||||||
metricDetailMap = metricDetailDTOList.stream().collect(Collectors.toMap(MetricDetailDTO::getMetric, Function.identity()));
|
|
||||||
}
|
|
||||||
|
|
||||||
//转换metricsSetMap
|
|
||||||
if (metricsSetMap != null && !metricsSetMap.isEmpty()) {
|
|
||||||
for (Map.Entry<String, Boolean> metricAndShowEntry : metricsSetMap.entrySet()) {
|
|
||||||
if (metricDetailMap.containsKey(metricAndShowEntry.getKey())) continue;
|
|
||||||
metricDetailMap.put(metricAndShowEntry.getKey(), new MetricDetailDTO(metricAndShowEntry.getKey(), metricAndShowEntry.getValue(), null));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (metricDetailMap.isEmpty()) {
|
|
||||||
return Result.buildSuc();
|
|
||||||
}
|
|
||||||
|
|
||||||
Set<UserMetricConfig> userMetricConfigs = getUserMetricConfig(operator);
|
|
||||||
for (MetricDetailDTO metricDetailDTO : metricDetailMap.values()) {
|
|
||||||
UserMetricConfig userMetricConfig = new UserMetricConfig(type, metricDetailDTO.getMetric(), metricDetailDTO.getSet(), metricDetailDTO.getRank());
|
|
||||||
userMetricConfigs.remove(userMetricConfig);
|
|
||||||
userMetricConfigs.add(userMetricConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
ConfigDTO configDTO = new ConfigDTO();
|
|
||||||
configDTO.setValueGroup(CONFIG_GROUP);
|
|
||||||
configDTO.setValueName(operator);
|
|
||||||
configDTO.setValue(JSON.toJSONString(userMetricConfigs));
|
|
||||||
configDTO.setOperator(operator);
|
|
||||||
configDTO.setStatus(1);
|
|
||||||
|
|
||||||
com.didiglobal.logi.security.common.Result<Void> result = configService.editConfig(configDTO, operator);
|
|
||||||
|
|
||||||
|
|
||||||
LOGGER.debug("method=updateUserMetricItem||clusterId={}||type={}||operator={}||userMetricConfigs={}||metricsSetMap={}",
|
|
||||||
clusterId, type, operator, JSON.toJSONString(userMetricConfigs), JSON.toJSONString(metricsSetMap));
|
|
||||||
|
|
||||||
return Result.build(result.successed());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private String itemSupportDesc(VersionControlItem item, boolean support){
|
|
||||||
if(support){return item.getDesc();}
|
|
||||||
|
|
||||||
boolean bMaxVersion = (item.getMaxVersion() == V_MAX.getVersionL().longValue());
|
|
||||||
|
|
||||||
String minVersion = VersionUtil.dNormailze(item.getMinVersion());
|
|
||||||
String maxVersion = VersionUtil.dNormailze(item.getMaxVersion());
|
|
||||||
|
|
||||||
if(bMaxVersion){
|
|
||||||
return item.getDesc() + String.format(NOT_SUPPORT_DESC, minVersion);
|
|
||||||
}
|
|
||||||
|
|
||||||
return item.getDesc() + String.format(NOT_SUPPORT_DESC1, minVersion, maxVersion);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<UserMetricConfig> getUserMetricConfig(String operator){
|
|
||||||
String value = configService.stringSetting(CONFIG_GROUP, operator, "");
|
|
||||||
if(StringUtils.isEmpty(value)){
|
|
||||||
return defaultMetrics;
|
|
||||||
}
|
|
||||||
|
|
||||||
return JSON.parseObject(value, new TypeReference<Set<UserMetricConfig>>() {});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void main(String[] args){
|
|
||||||
Set<UserMetricConfig> defaultMetrics = new HashSet<>();
|
|
||||||
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_IN, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_MESSAGES, true));
|
|
||||||
defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_MESSAGES, true));
|
|
||||||
|
|
||||||
String value = JSON.toJSONString(defaultMetrics);
|
|
||||||
|
|
||||||
Set<UserMetricConfig> userMetricConfigs = JSON.parseObject(value, new TypeReference<Set<UserMetricConfig>>(){});
|
|
||||||
|
|
||||||
System.out.println(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<artifactId>km-collector</artifactId>
|
|
||||||
<version>${revision}</version>
|
|
||||||
<packaging>jar</packaging>
|
|
||||||
|
|
||||||
<parent>
|
|
||||||
<artifactId>km</artifactId>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<version>${revision}</version>
|
|
||||||
</parent>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<artifactId>km-common</artifactId>
|
|
||||||
<version>${project.parent.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.xiaojukeji.kafka</groupId>
|
|
||||||
<artifactId>km-core</artifactId>
|
|
||||||
<version>${project.parent.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework</groupId>
|
|
||||||
<artifactId>spring-webmvc</artifactId>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
|
||||||
</project>
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric;
|
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.service.CollectThreadPoolService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BaseMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.component.SpringTool;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
public abstract class AbstractMetricCollector<M, C> {
|
|
||||||
public abstract String getClusterVersion(C c);
|
|
||||||
|
|
||||||
public abstract VersionItemTypeEnum collectorType();
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private CollectThreadPoolService collectThreadPoolService;
|
|
||||||
|
|
||||||
public abstract void collectMetrics(C c);
|
|
||||||
|
|
||||||
protected FutureWaitUtil<Void> getFutureUtilByClusterPhyId(Long clusterPhyId) {
|
|
||||||
return collectThreadPoolService.selectSuitableFutureUtil(clusterPhyId * 1000L + this.collectorType().getCode());
|
|
||||||
}
|
|
||||||
|
|
||||||
protected <T extends BaseMetricEvent> void publishMetric(T event){
|
|
||||||
SpringTool.publish(event);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.connect;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
public abstract class AbstractConnectMetricCollector<M> extends AbstractMetricCollector<M, ConnectCluster> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(AbstractConnectMetricCollector.class);
|
|
||||||
|
|
||||||
protected static final ILog METRIC_COLLECTED_LOGGER = LoggerUtil.getMetricCollectedLogger();
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectClusterService connectClusterService;
|
|
||||||
|
|
||||||
public abstract List<M> collectConnectMetrics(ConnectCluster connectCluster);
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getClusterVersion(ConnectCluster connectCluster){
|
|
||||||
return connectClusterService.getClusterVersion(connectCluster.getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void collectMetrics(ConnectCluster connectCluster) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
// 采集指标
|
|
||||||
List<M> metricsList = this.collectConnectMetrics(connectCluster);
|
|
||||||
|
|
||||||
// 输出耗时信息
|
|
||||||
LOGGER.info(
|
|
||||||
"metricType={}||connectClusterId={}||costTimeUnitMs={}",
|
|
||||||
this.collectorType().getMessage(), connectCluster.getId(), System.currentTimeMillis() - startTime
|
|
||||||
);
|
|
||||||
|
|
||||||
// 输出采集到的指标信息
|
|
||||||
METRIC_COLLECTED_LOGGER.debug("metricType={}||connectClusterId={}||metrics={}!",
|
|
||||||
this.collectorType().getMessage(), connectCluster.getId(), ConvertUtil.obj2Json(metricsList)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.connect;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectClusterMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectClusterMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CONNECT_CLUSTER;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class ConnectClusterMetricCollector extends AbstractConnectMetricCollector<ConnectClusterMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(ConnectClusterMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectClusterMetricService connectClusterMetricService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ConnectClusterMetrics> collectConnectMetrics(ConnectCluster connectCluster) {
|
|
||||||
Long startTime = System.currentTimeMillis();
|
|
||||||
Long clusterPhyId = connectCluster.getKafkaClusterPhyId();
|
|
||||||
Long connectClusterId = connectCluster.getId();
|
|
||||||
|
|
||||||
ConnectClusterMetrics metrics = new ConnectClusterMetrics(clusterPhyId, connectClusterId);
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(getClusterVersion(connectCluster), collectorType().getCode());
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(connectClusterId);
|
|
||||||
|
|
||||||
for (VersionControlItem item : items) {
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=ConnectClusterMetricCollector||connectClusterId=%d||metricName=%s", connectClusterId, item.getName()),
|
|
||||||
30000,
|
|
||||||
() -> {
|
|
||||||
try {
|
|
||||||
Result<ConnectClusterMetrics> ret = connectClusterMetricService.collectConnectClusterMetricsFromKafka(connectClusterId, item.getName());
|
|
||||||
if (null == ret || !ret.hasData()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
metrics.putMetric(ret.getData().getMetrics());
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectConnectMetrics||connectClusterId={}||metricName={}||errMsg=exception!",
|
|
||||||
connectClusterId, item.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
future.waitExecute(30000);
|
|
||||||
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
|
|
||||||
this.publishMetric(new ConnectClusterMetricEvent(this, Collections.singletonList(metrics)));
|
|
||||||
|
|
||||||
return Collections.singletonList(metrics);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_CONNECT_CLUSTER;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.connect;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectorMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectorMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.connect.ConnectorTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CONNECT_CONNECTOR;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class ConnectConnectorMetricCollector extends AbstractConnectMetricCollector<ConnectorMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(ConnectConnectorMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorService connectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorMetricService connectorMetricService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ConnectorMetrics> collectConnectMetrics(ConnectCluster connectCluster) {
|
|
||||||
Long clusterPhyId = connectCluster.getKafkaClusterPhyId();
|
|
||||||
Long connectClusterId = connectCluster.getId();
|
|
||||||
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(connectCluster), collectorType().getCode());
|
|
||||||
Result<List<String>> connectorList = connectorService.listConnectorsFromCluster(connectCluster);
|
|
||||||
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(connectClusterId);
|
|
||||||
|
|
||||||
List<ConnectorMetrics> metricsList = new ArrayList<>();
|
|
||||||
for (String connectorName : connectorList.getData()) {
|
|
||||||
ConnectorMetrics metrics = new ConnectorMetrics(connectClusterId, connectorName);
|
|
||||||
metrics.setClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
metricsList.add(metrics);
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=ConnectConnectorMetricCollector||connectClusterId=%d||connectorName=%s", connectClusterId, connectorName),
|
|
||||||
30000,
|
|
||||||
() -> collectMetrics(connectClusterId, connectorName, metrics, items)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
future.waitResult(30000);
|
|
||||||
|
|
||||||
this.publishMetric(new ConnectorMetricEvent(this, metricsList));
|
|
||||||
|
|
||||||
return metricsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_CONNECT_CONNECTOR;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private void collectMetrics(Long connectClusterId, String connectorName, ConnectorMetrics metrics, List<VersionControlItem> items) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
ConnectorTypeEnum connectorType = connectorService.getConnectorType(connectClusterId, connectorName);
|
|
||||||
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
|
|
||||||
for (VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
//过滤已测得指标
|
|
||||||
if (metrics.getMetrics().get(v.getName()) != null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<ConnectorMetrics> ret = connectorMetricService.collectConnectClusterMetricsFromKafka(connectClusterId, connectorName, v.getName(), connectorType);
|
|
||||||
if (null == ret || ret.failed() || null == ret.getData()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics.putMetric(ret.getData().getMetrics());
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectMetrics||connectClusterId={}||connectorName={}||metric={}||errMsg=exception!",
|
|
||||||
connectClusterId, connectorName, v.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录采集性能
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,117 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.connect.mm2;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.metric.connect.AbstractConnectMetricCollector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.mm2.MirrorMakerTopic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.mm2.MirrorMakerMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.mm2.MirrorMakerMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.connect.ConnectorPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.mm2.MirrorMakerMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.connect.mm2.MirrorMakerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.constant.connect.KafkaConnectConstant.MIRROR_MAKER_SOURCE_CONNECTOR_TYPE;
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CONNECT_MIRROR_MAKER;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/12/15
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class MirrorMakerMetricCollector extends AbstractConnectMetricCollector<MirrorMakerMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(MirrorMakerMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MirrorMakerService mirrorMakerService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ConnectorService connectorService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MirrorMakerMetricService mirrorMakerMetricService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_CONNECT_MIRROR_MAKER;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<MirrorMakerMetrics> collectConnectMetrics(ConnectCluster connectCluster) {
|
|
||||||
Long clusterPhyId = connectCluster.getKafkaClusterPhyId();
|
|
||||||
Long connectClusterId = connectCluster.getId();
|
|
||||||
|
|
||||||
List<ConnectorPO> mirrorMakerList = connectorService.listByConnectClusterIdFromDB(connectClusterId).stream().filter(elem -> elem.getConnectorClassName().equals(MIRROR_MAKER_SOURCE_CONNECTOR_TYPE)).collect(Collectors.toList());
|
|
||||||
Map<String, MirrorMakerTopic> mirrorMakerTopicMap = mirrorMakerService.getMirrorMakerTopicMap(connectClusterId).getData();
|
|
||||||
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(connectCluster), collectorType().getCode());
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
List<MirrorMakerMetrics> metricsList = new ArrayList<>();
|
|
||||||
|
|
||||||
for (ConnectorPO mirrorMaker : mirrorMakerList) {
|
|
||||||
MirrorMakerMetrics metrics = new MirrorMakerMetrics(clusterPhyId, connectClusterId, mirrorMaker.getConnectorName());
|
|
||||||
metricsList.add(metrics);
|
|
||||||
|
|
||||||
List<MirrorMakerTopic> mirrorMakerTopicList = mirrorMakerService.getMirrorMakerTopicList(mirrorMaker, mirrorMakerTopicMap);
|
|
||||||
future.runnableTask(String.format("class=MirrorMakerMetricCollector||connectClusterId=%d||mirrorMakerName=%s", connectClusterId, mirrorMaker.getConnectorName()),
|
|
||||||
30000,
|
|
||||||
() -> collectMetrics(connectClusterId, mirrorMaker.getConnectorName(), metrics, items, mirrorMakerTopicList));
|
|
||||||
}
|
|
||||||
future.waitResult(30000);
|
|
||||||
|
|
||||||
this.publishMetric(new MirrorMakerMetricEvent(this,metricsList));
|
|
||||||
|
|
||||||
return metricsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
private void collectMetrics(Long connectClusterId, String mirrorMakerName, MirrorMakerMetrics metrics, List<VersionControlItem> items, List<MirrorMakerTopic> mirrorMakerTopicList) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
|
|
||||||
for (VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
//已测量指标过滤
|
|
||||||
if (metrics.getMetrics().get(v.getName()) != null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<MirrorMakerMetrics> ret = mirrorMakerMetricService.collectMirrorMakerMetricsFromKafka(connectClusterId, mirrorMakerName, mirrorMakerTopicList, v.getName());
|
|
||||||
if (ret == null || !ret.hasData()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
metrics.putMetric(ret.getData().getMetrics());
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectMetrics||connectClusterId={}||mirrorMakerName={}||metric={}||errMsg=exception!",
|
|
||||||
connectClusterId, mirrorMakerName, v.getName(), e
|
|
||||||
);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
public abstract class AbstractKafkaMetricCollector<M> extends AbstractMetricCollector<M, ClusterPhy> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(AbstractMetricCollector.class);
|
|
||||||
|
|
||||||
protected static final ILog METRIC_COLLECTED_LOGGER = LoggerUtil.getMetricCollectedLogger();
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterPhyService clusterPhyService;
|
|
||||||
|
|
||||||
public abstract List<M> collectKafkaMetrics(ClusterPhy clusterPhy);
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getClusterVersion(ClusterPhy clusterPhy){
|
|
||||||
return clusterPhyService.getVersionFromCacheFirst(clusterPhy.getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void collectMetrics(ClusterPhy clusterPhy) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
// 采集指标
|
|
||||||
List<M> metricsList = this.collectKafkaMetrics(clusterPhy);
|
|
||||||
|
|
||||||
// 输出耗时信息
|
|
||||||
LOGGER.info(
|
|
||||||
"metricType={}||clusterPhyId={}||costTimeUnitMs={}",
|
|
||||||
this.collectorType().getMessage(), clusterPhy.getId(), System.currentTimeMillis() - startTime
|
|
||||||
);
|
|
||||||
|
|
||||||
// 输出采集到的指标信息
|
|
||||||
METRIC_COLLECTED_LOGGER.debug("metricType={}||clusterPhyId={}||metrics={}!",
|
|
||||||
this.collectorType().getMessage(), clusterPhy.getId(), ConvertUtil.obj2Json(metricsList)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,102 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BrokerMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_BROKER;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class BrokerMetricCollector extends AbstractKafkaMetricCollector<BrokerMetrics> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(BrokerMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerMetricService brokerMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private BrokerService brokerService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<BrokerMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
|
|
||||||
Long clusterPhyId = clusterPhy.getId();
|
|
||||||
|
|
||||||
List<Broker> brokers = brokerService.listAliveBrokersFromDB(clusterPhy.getId());
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
|
|
||||||
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
List<BrokerMetrics> metricsList = new ArrayList<>();
|
|
||||||
for(Broker broker : brokers) {
|
|
||||||
BrokerMetrics metrics = new BrokerMetrics(clusterPhyId, broker.getBrokerId(), broker.getHost(), broker.getPort());
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
metricsList.add(metrics);
|
|
||||||
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=BrokerMetricCollector||clusterPhyId=%d||brokerId=%d", clusterPhyId, broker.getBrokerId()),
|
|
||||||
30000,
|
|
||||||
() -> collectMetrics(clusterPhyId, metrics, items)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
future.waitExecute(30000);
|
|
||||||
this.publishMetric(new BrokerMetricEvent(this, metricsList));
|
|
||||||
|
|
||||||
return metricsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_BROKER;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private void collectMetrics(Long clusterPhyId, BrokerMetrics metrics, List<VersionControlItem> items) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
for(VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
if(metrics.getMetrics().containsKey(v.getName())) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<BrokerMetrics> ret = brokerMetricService.collectBrokerMetricsFromKafkaWithCacheFirst(clusterPhyId, metrics.getBrokerId(), v.getName());
|
|
||||||
if(null == ret || ret.failed() || null == ret.getData()){
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics.putMetric(ret.getData().getMetrics());
|
|
||||||
} catch (Exception e){
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectMetrics||clusterPhyId={}||brokerId={}||metricName={}||errMsg=exception!",
|
|
||||||
clusterPhyId, metrics.getBrokerId(), v.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录采集性能
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.ClusterMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CLUSTER;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class ClusterMetricCollector extends AbstractKafkaMetricCollector<ClusterMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(ClusterMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ClusterMetricService clusterMetricService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ClusterMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
|
|
||||||
Long startTime = System.currentTimeMillis();
|
|
||||||
Long clusterPhyId = clusterPhy.getId();
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
|
|
||||||
|
|
||||||
ClusterMetrics metrics = new ClusterMetrics(clusterPhyId, clusterPhy.getKafkaVersion());
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
for(VersionControlItem v : items) {
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=ClusterMetricCollector||clusterPhyId=%d||metricName=%s", clusterPhyId, v.getName()),
|
|
||||||
30000,
|
|
||||||
() -> {
|
|
||||||
try {
|
|
||||||
if(null != metrics.getMetrics().get(v.getName())){
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<ClusterMetrics> ret = clusterMetricService.collectClusterMetricsFromKafka(clusterPhyId, v.getName());
|
|
||||||
if(null == ret || ret.failed() || null == ret.getData()){
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics.putMetric(ret.getData().getMetrics());
|
|
||||||
} catch (Exception e){
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectKafkaMetrics||clusterPhyId={}||metricName={}||errMsg=exception!",
|
|
||||||
clusterPhyId, v.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
future.waitExecute(30000);
|
|
||||||
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
|
|
||||||
publishMetric(new ClusterMetricEvent(this, Collections.singletonList(metrics)));
|
|
||||||
|
|
||||||
return Collections.singletonList(metrics);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_CLUSTER;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.GroupMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.apache.kafka.common.TopicPartition;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_GROUP;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class GroupMetricCollector extends AbstractKafkaMetricCollector<GroupMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(GroupMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupMetricService groupMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private GroupService groupService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<GroupMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
|
|
||||||
Long clusterPhyId = clusterPhy.getId();
|
|
||||||
|
|
||||||
List<String> groupNameList = new ArrayList<>();
|
|
||||||
try {
|
|
||||||
groupNameList = groupService.listGroupsFromKafka(clusterPhy);
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error("method=collectKafkaMetrics||clusterPhyId={}||msg=exception!", clusterPhyId, e);
|
|
||||||
}
|
|
||||||
|
|
||||||
if(ValidateUtils.isEmptyList(groupNameList)) {
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
|
|
||||||
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
Map<String, List<GroupMetrics>> metricsMap = new ConcurrentHashMap<>();
|
|
||||||
for(String groupName : groupNameList) {
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=GroupMetricCollector||clusterPhyId=%d||groupName=%s", clusterPhyId, groupName),
|
|
||||||
30000,
|
|
||||||
() -> collectMetrics(clusterPhyId, groupName, metricsMap, items));
|
|
||||||
}
|
|
||||||
|
|
||||||
future.waitResult(30000);
|
|
||||||
|
|
||||||
List<GroupMetrics> metricsList = metricsMap.values().stream().collect(ArrayList::new, ArrayList::addAll, ArrayList::addAll);
|
|
||||||
|
|
||||||
publishMetric(new GroupMetricEvent(this, metricsList));
|
|
||||||
return metricsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_GROUP;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private void collectMetrics(Long clusterPhyId, String groupName, Map<String, List<GroupMetrics>> metricsMap, List<VersionControlItem> items) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
Map<TopicPartition, GroupMetrics> subMetricMap = new HashMap<>();
|
|
||||||
|
|
||||||
GroupMetrics groupMetrics = new GroupMetrics(clusterPhyId, groupName, true);
|
|
||||||
groupMetrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
|
|
||||||
for(VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
String metricName = v.getName();
|
|
||||||
|
|
||||||
Result<List<GroupMetrics>> ret = groupMetricService.collectGroupMetricsFromKafka(clusterPhyId, groupName, metricName);
|
|
||||||
if(null == ret || ret.failed() || ValidateUtils.isEmptyList(ret.getData())) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret.getData().forEach(metrics -> {
|
|
||||||
if (metrics.isBGroupMetric()) {
|
|
||||||
groupMetrics.putMetric(metrics.getMetrics());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
TopicPartition tp = new TopicPartition(metrics.getTopic(), metrics.getPartitionId());
|
|
||||||
subMetricMap.putIfAbsent(tp, new GroupMetrics(clusterPhyId, metrics.getPartitionId(), metrics.getTopic(), groupName, false));
|
|
||||||
subMetricMap.get(tp).putMetric(metrics.getMetrics());
|
|
||||||
});
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectMetrics||clusterPhyId={}||groupName={}||errMsg=exception!",
|
|
||||||
clusterPhyId, groupName, e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
List<GroupMetrics> metricsList = new ArrayList<>();
|
|
||||||
metricsList.add(groupMetrics);
|
|
||||||
metricsList.addAll(subMetricMap.values());
|
|
||||||
|
|
||||||
// 记录采集性能
|
|
||||||
groupMetrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
|
|
||||||
metricsMap.put(groupName, metricsList);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.PartitionMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_PARTITION;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class PartitionMetricCollector extends AbstractKafkaMetricCollector<PartitionMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(PartitionMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PartitionMetricService partitionMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<PartitionMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
|
|
||||||
Long clusterPhyId = clusterPhy.getId();
|
|
||||||
List<Topic> topicList = topicService.listTopicsFromCacheFirst(clusterPhyId);
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
|
|
||||||
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
Map<String, Map<Integer, PartitionMetrics>> metricsMap = new ConcurrentHashMap<>();
|
|
||||||
for (Topic topic : topicList) {
|
|
||||||
metricsMap.put(topic.getTopicName(), new ConcurrentHashMap<>());
|
|
||||||
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=PartitionMetricCollector||clusterPhyId=%d||topicName=%s", clusterPhyId, topic.getTopicName()),
|
|
||||||
30000,
|
|
||||||
() -> this.collectMetrics(clusterPhyId, topic.getTopicName(), metricsMap.get(topic.getTopicName()), items)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
future.waitExecute(30000);
|
|
||||||
|
|
||||||
List<PartitionMetrics> metricsList = new ArrayList<>();
|
|
||||||
metricsMap.values().forEach(elem -> metricsList.addAll(elem.values()));
|
|
||||||
|
|
||||||
this.publishMetric(new PartitionMetricEvent(this, metricsList));
|
|
||||||
|
|
||||||
return metricsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_PARTITION;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private void collectMetrics(Long clusterPhyId, String topicName, Map<Integer, PartitionMetrics> metricsMap, List<VersionControlItem> items) {
|
|
||||||
Set<String> collectedMetricsNameSet = new HashSet<>();
|
|
||||||
for (VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
if (collectedMetricsNameSet.contains(v.getName())) {
|
|
||||||
// 指标已存在
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
collectedMetricsNameSet.add(v.getName());
|
|
||||||
|
|
||||||
Result<List<PartitionMetrics>> ret = partitionMetricService.collectPartitionsMetricsFromKafkaWithCache(
|
|
||||||
clusterPhyId,
|
|
||||||
topicName,
|
|
||||||
v.getName()
|
|
||||||
);
|
|
||||||
if (null == ret || ret.failed() || null == ret.getData() || ret.getData().isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录已经采集的指标
|
|
||||||
collectedMetricsNameSet.addAll(ret.getData().get(0).getMetrics().keySet());
|
|
||||||
|
|
||||||
// 放到map中
|
|
||||||
for (PartitionMetrics subMetrics: ret.getData()) {
|
|
||||||
metricsMap.putIfAbsent(subMetrics.getPartitionId(), subMetrics);
|
|
||||||
PartitionMetrics allMetrics = metricsMap.get(subMetrics.getPartitionId());
|
|
||||||
allMetrics.putMetric(subMetrics.getMetrics());
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.info(
|
|
||||||
"method=collectMetrics||clusterPhyId={}||topicName={}||metricName={}||errMsg=exception",
|
|
||||||
clusterPhyId, topicName, v.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.TopicMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_TOPIC;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class TopicMetricCollector extends AbstractKafkaMetricCollector<TopicMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(TopicMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicService topicService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private TopicMetricService topicMetricService;
|
|
||||||
|
|
||||||
private static final Integer AGG_METRICS_BROKER_ID = -10000;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<TopicMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
|
|
||||||
Long clusterPhyId = clusterPhy.getId();
|
|
||||||
List<Topic> topics = topicService.listTopicsFromCacheFirst(clusterPhyId);
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
|
|
||||||
|
|
||||||
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);
|
|
||||||
|
|
||||||
Map<String/*Topic名称*/, Map<Integer/*BrokerId*/, TopicMetrics/*metrics*/>> allMetricsMap = new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
for(Topic topic : topics) {
|
|
||||||
Map<Integer, TopicMetrics> metricsMap = new ConcurrentHashMap<>();
|
|
||||||
metricsMap.put(AGG_METRICS_BROKER_ID, new TopicMetrics(topic.getTopicName(), clusterPhyId));
|
|
||||||
metricsMap.get(AGG_METRICS_BROKER_ID).putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
|
|
||||||
allMetricsMap.put(topic.getTopicName(), metricsMap);
|
|
||||||
|
|
||||||
future.runnableTask(
|
|
||||||
String.format("class=TopicMetricCollector||clusterPhyId=%d||topicName=%s", clusterPhyId, topic.getTopicName()),
|
|
||||||
30000,
|
|
||||||
() -> collectMetrics(clusterPhyId, topic.getTopicName(), metricsMap, items)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
future.waitExecute(30000);
|
|
||||||
|
|
||||||
List<TopicMetrics> metricsList = new ArrayList<>();
|
|
||||||
allMetricsMap.values().forEach(elem -> metricsList.addAll(elem.values()));
|
|
||||||
|
|
||||||
this.publishMetric(new TopicMetricEvent(this, metricsList));
|
|
||||||
|
|
||||||
return metricsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_TOPIC;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private void collectMetrics(Long clusterPhyId, String topicName, Map<Integer, TopicMetrics> metricsMap, List<VersionControlItem> items) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
TopicMetrics aggMetrics = metricsMap.get(AGG_METRICS_BROKER_ID);
|
|
||||||
for (VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
if (aggMetrics.getMetrics().containsKey(v.getName())) {
|
|
||||||
// 如果已经有该指标，则直接continue
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Result<List<TopicMetrics>> ret = topicMetricService.collectTopicMetricsFromKafkaWithCacheFirst(clusterPhyId, topicName, v.getName());
|
|
||||||
if (null == ret || ret.failed() || ValidateUtils.isEmptyList(ret.getData())) {
|
|
||||||
// 返回为空、错误、无数据的情况下，直接跳过
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录数据
|
|
||||||
ret.getData().stream().forEach(metrics -> {
|
|
||||||
if (metrics.isBBrokerAgg()) {
|
|
||||||
aggMetrics.putMetric(metrics.getMetrics());
|
|
||||||
} else {
|
|
||||||
metricsMap.putIfAbsent(
|
|
||||||
metrics.getBrokerId(),
|
|
||||||
new TopicMetrics(topicName, clusterPhyId, metrics.getBrokerId(), false)
|
|
||||||
);
|
|
||||||
|
|
||||||
metricsMap.get(metrics.getBrokerId()).putMetric(metrics.getMetrics());
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectMetrics||clusterPhyId={}||topicName={}||metricName={}||errMsg=exception!",
|
|
||||||
clusterPhyId, topicName, v.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录采集性能
|
|
||||||
aggMetrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.ZKConfig;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.ZookeeperMetricParam;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.ZookeeperMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.zookeeper.ZookeeperInfo;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ZookeeperMetrics;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.zookeeper.ZookeeperMetricService;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.zookeeper.ZookeeperService;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_ZOOKEEPER;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author didi
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class ZookeeperMetricCollector extends AbstractKafkaMetricCollector<ZookeeperMetrics> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(ZookeeperMetricCollector.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VersionControlService versionControlService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ZookeeperMetricService zookeeperMetricService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ZookeeperService zookeeperService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KafkaControllerService kafkaControllerService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<ZookeeperMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
|
|
||||||
Long startTime = System.currentTimeMillis();
|
|
||||||
Long clusterPhyId = clusterPhy.getId();
|
|
||||||
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
|
|
||||||
List<ZookeeperInfo> aliveZKList = zookeeperService.listFromDBByCluster(clusterPhyId)
|
|
||||||
.stream()
|
|
||||||
.filter(elem -> Constant.ALIVE.equals(elem.getStatus()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
KafkaController kafkaController = kafkaControllerService.getKafkaControllerFromDB(clusterPhyId);
|
|
||||||
|
|
||||||
ZookeeperMetrics metrics = ZookeeperMetrics.initWithMetric(clusterPhyId, Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
|
|
||||||
if (ValidateUtils.isEmptyList(aliveZKList)) {
|
|
||||||
// 没有存活的ZK时，发布事件，然后直接返回
|
|
||||||
publishMetric(new ZookeeperMetricEvent(this, Collections.singletonList(metrics)));
|
|
||||||
return Collections.singletonList(metrics);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 构造参数
|
|
||||||
ZookeeperMetricParam param = new ZookeeperMetricParam(
|
|
||||||
clusterPhyId,
|
|
||||||
aliveZKList.stream().map(elem -> new Tuple<String, Integer>(elem.getHost(), elem.getPort())).collect(Collectors.toList()),
|
|
||||||
ConvertUtil.str2ObjByJson(clusterPhy.getZkProperties(), ZKConfig.class),
|
|
||||||
kafkaController == null? Constant.INVALID_CODE: kafkaController.getBrokerId(),
|
|
||||||
null
|
|
||||||
);
|
|
||||||
|
|
||||||
for(VersionControlItem v : items) {
|
|
||||||
try {
|
|
||||||
if(null != metrics.getMetrics().get(v.getName())) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
param.setMetricName(v.getName());
|
|
||||||
|
|
||||||
Result<ZookeeperMetrics> ret = zookeeperMetricService.collectMetricsFromZookeeper(param);
|
|
||||||
if(null == ret || ret.failed() || null == ret.getData()){
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics.putMetric(ret.getData().getMetrics());
|
|
||||||
} catch (Exception e){
|
|
||||||
LOGGER.error(
|
|
||||||
"method=collectMetrics||clusterPhyId={}||metricName={}||errMsg=exception!",
|
|
||||||
clusterPhyId, v.getName(), e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
|
|
||||||
|
|
||||||
this.publishMetric(new ZookeeperMetricEvent(this, Collections.singletonList(metrics)));
|
|
||||||
|
|
||||||
return Collections.singletonList(metrics);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public VersionItemTypeEnum collectorType() {
|
|
||||||
return METRIC_ZOOKEEPER;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,263 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.service;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.github.benmanes.caffeine.cache.Cache;
|
|
||||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.*;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class CollectThreadPoolService {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(CollectThreadPoolService.class);
|
|
||||||
|
|
||||||
private final AtomicLong shardIdx = new AtomicLong(0L);
|
|
||||||
|
|
||||||
@Value(value = "${thread-pool.collector.future-util.num:1}")
|
|
||||||
private Integer futureUtilNum;
|
|
||||||
|
|
||||||
@Value(value = "${thread-pool.collector.future-util.thread-num:8}")
|
|
||||||
private Integer futureUtilThreadNum;
|
|
||||||
|
|
||||||
@Value(value = "${thread-pool.collector.future-util.queue-size:10000}")
|
|
||||||
private Integer futureUtilQueueSize;
|
|
||||||
|
|
||||||
@Value(value = "${thread-pool.collector.future-util.select-suitable-enable:true}")
|
|
||||||
private Boolean futureUtilSelectSuitableEnable;
|
|
||||||
|
|
||||||
@Value(value = "${thread-pool.collector.future-util.suitable-queue-size:5000}")
|
|
||||||
private Integer futureUtilSuitableQueueSize;
|
|
||||||
|
|
||||||
private static final Map<Long, FutureWaitUtil<Void>> SHARD_ID_FUTURE_UTIL_MAP = new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
private static final Cache<Long, Long> PHYSICAL_CLUSTER_ID_SHARD_ID_CACHE = Caffeine
|
|
||||||
.newBuilder()
|
|
||||||
.expireAfterWrite(16, TimeUnit.MINUTES)
|
|
||||||
.maximumSize(1000)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
private void init() {
|
|
||||||
if (futureUtilNum <= 0) {
|
|
||||||
futureUtilNum = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 初始化job线程池
|
|
||||||
for (int idx = 0; idx < futureUtilNum; ++idx) {
|
|
||||||
closeOldAndCreateNew((long)idx);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public FutureWaitUtil<Void> selectSuitableFutureUtil(Long clusterPhyId) {
|
|
||||||
// 获取集群对应的shardId
|
|
||||||
Long shardId = this.getShardId(clusterPhyId);
|
|
||||||
|
|
||||||
return SHARD_ID_FUTURE_UTIL_MAP.get(shardId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** private method ****************************************************/
|
|
||||||
|
|
||||||
private Long getShardId(Long clusterPhyId) {
|
|
||||||
Long shardId = PHYSICAL_CLUSTER_ID_SHARD_ID_CACHE.getIfPresent(clusterPhyId);
|
|
||||||
if (shardId == null) {
|
|
||||||
shardId = shardIdx.incrementAndGet() % this.futureUtilNum;
|
|
||||||
}
|
|
||||||
|
|
||||||
PHYSICAL_CLUSTER_ID_SHARD_ID_CACHE.put(clusterPhyId, shardId);
|
|
||||||
return shardId;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**************************************************** schedule flush method ****************************************************/
|
|
||||||
|
|
||||||
@Scheduled(cron="0 0/5 * * * ?")
|
|
||||||
public void flush() {
|
|
||||||
// 每个shard对应的集群ID，这里使用cache的原因是，需要将长期不使用的集群过滤掉
|
|
||||||
Map<Long, List<Long>> shardIdPhysicalClusterIdListMap = new HashMap<>();
|
|
||||||
for (Map.Entry<Long, Long> entry: PHYSICAL_CLUSTER_ID_SHARD_ID_CACHE.asMap().entrySet()) {
|
|
||||||
shardIdPhysicalClusterIdListMap.putIfAbsent(entry.getValue(), new ArrayList<>());
|
|
||||||
shardIdPhysicalClusterIdListMap.get(entry.getValue()).add(entry.getKey());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 集群在线程池的分布信息
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
for (Map.Entry<Long, FutureWaitUtil<Void>> entry: SHARD_ID_FUTURE_UTIL_MAP.entrySet()) {
|
|
||||||
// 释放被canceled的任务
|
|
||||||
entry.getValue().purgeExecutor();
|
|
||||||
|
|
||||||
sb.append("shardId:").append(entry.getKey());
|
|
||||||
sb.append(" queueSize:").append(entry.getValue().getExecutorQueueSize());
|
|
||||||
sb.append(" physicalClusterIdList:").append(
|
|
||||||
CommonUtils.longList2String(shardIdPhysicalClusterIdListMap.getOrDefault(entry.getKey(), new ArrayList<>()))
|
|
||||||
);
|
|
||||||
sb.append("\t\t\t");
|
|
||||||
if (entry.getValue().getExecutorQueueSize() >= this.futureUtilSuitableQueueSize) {
|
|
||||||
LOGGER.info("JobThreadPoolInfo\t\t\t shardId:{} queueSize:{} physicalClusterIdList:{}.",
|
|
||||||
entry.getKey(),
|
|
||||||
entry.getValue().getExecutorQueueSize(),
|
|
||||||
CommonUtils.longList2String(shardIdPhysicalClusterIdListMap.getOrDefault(entry.getKey(), new ArrayList<>()))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
LOGGER.info("JobThreadPoolInfo\t\t\t {}...", sb);
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (futureUtilSelectSuitableEnable != null && futureUtilSelectSuitableEnable) {
|
|
||||||
reBalancePhysicalClusterShard(shardIdPhysicalClusterIdListMap);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error("rebalance job-thread-pool failed.", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void reBalancePhysicalClusterShard(Map<Long, List<Long>> shardIdPhysicalClusterIdListMap) {
|
|
||||||
List<Long> withoutClusterShardIdList = new ArrayList<>(); // 无集群任务的线程池
|
|
||||||
List<Long> idleShardIdList = new ArrayList<>(); // 空闲的线程池
|
|
||||||
List<Long> notBusyShardIdList = new ArrayList<>(); // 不忙的线程池
|
|
||||||
List<Long> busyShardIdList = new ArrayList<>(); // 忙的线程池
|
|
||||||
List<Long> overflowShardIdList = new ArrayList<>(); // 已处理不过来的线程池
|
|
||||||
|
|
||||||
// 统计各类线程池信息
|
|
||||||
for (Map.Entry<Long, List<Long>> entry: shardIdPhysicalClusterIdListMap.entrySet()) {
|
|
||||||
Integer queueSize = SHARD_ID_FUTURE_UTIL_MAP.get(entry.getKey()).getExecutorQueueSize();
|
|
||||||
if (entry.getValue().isEmpty()) {
|
|
||||||
withoutClusterShardIdList.add(entry.getKey());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (queueSize == 0) {
|
|
||||||
// 队列为空
|
|
||||||
idleShardIdList.add(entry.getKey());
|
|
||||||
} else if (queueSize <= futureUtilSuitableQueueSize) {
|
|
||||||
// 队列较空闲
|
|
||||||
notBusyShardIdList.add(entry.getKey());
|
|
||||||
} else if (queueSize >= futureUtilSuitableQueueSize - 10) {
|
|
||||||
// 队列处理不过来
|
|
||||||
overflowShardIdList.add(entry.getKey());
|
|
||||||
} else {
|
|
||||||
// 队列忙
|
|
||||||
busyShardIdList.add(entry.getKey());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 将队列满的线程池的集群拆分到不同的线程池中
|
|
||||||
this.moveShardClusterToSuitableThreadPool(overflowShardIdList, shardIdPhysicalClusterIdListMap, withoutClusterShardIdList, idleShardIdList, notBusyShardIdList, true);
|
|
||||||
|
|
||||||
// 将busy队列的线程池的集群拆分到不同的线程池中
|
|
||||||
this.moveShardClusterToSuitableThreadPool(busyShardIdList, shardIdPhysicalClusterIdListMap, withoutClusterShardIdList, idleShardIdList, notBusyShardIdList, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void moveShardClusterToSuitableThreadPool(List<Long> needMoveShardIdList,
|
|
||||||
Map<Long, List<Long>> shardIdPhysicalClusterIdListMap,
|
|
||||||
List<Long> withoutClusterShardIdList,
|
|
||||||
List<Long> idleShardIdList,
|
|
||||||
List<Long> notBusyShardIdList,
|
|
||||||
boolean clearTaskIfFullAndOnlyOneCluster) {
|
|
||||||
for (Long needMoveShardId: needMoveShardIdList) {
|
|
||||||
List<Long> physicalClusterIdList = shardIdPhysicalClusterIdListMap.get(needMoveShardId);
|
|
||||||
if ((physicalClusterIdList == null || physicalClusterIdList.isEmpty() || physicalClusterIdList.size() == 1) && clearTaskIfFullAndOnlyOneCluster) {
|
|
||||||
// 仅一个集群，并且满了，则清空任务，重新跑任务
|
|
||||||
closeOldAndCreateNew(needMoveShardId);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (physicalClusterIdList == null) {
|
|
||||||
// 无集群
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int idx = 0; idx < physicalClusterIdList.size() - 1; ++idx) {
|
|
||||||
Long newSuitableShardId = this.selectAndEmptySuitableThreadPool(shardIdPhysicalClusterIdListMap, withoutClusterShardIdList, idleShardIdList, notBusyShardIdList);
|
|
||||||
if (newSuitableShardId == null) {
|
|
||||||
LOGGER.info("without suitable job-thread-pool and return.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
modifyPhysicalClusterIdAndShardIdCache(physicalClusterIdList.get(idx), newSuitableShardId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Long selectAndEmptySuitableThreadPool(Map<Long, List<Long>> shardIdPhysicalClusterIdListMap,
|
|
||||||
List<Long> withoutClusterShardIdList,
|
|
||||||
List<Long> idleShardIdList,
|
|
||||||
List<Long> notBusyShardIdList) {
|
|
||||||
if (!withoutClusterShardIdList.isEmpty()) {
|
|
||||||
// 先放入无集群任务的线程池
|
|
||||||
return withoutClusterShardIdList.remove((int) 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 上一条件不满足时，优先放入比较空闲的池子
|
|
||||||
Long newShardId = this.selectAndEmptySuitableThreadPool(shardIdPhysicalClusterIdListMap, idleShardIdList);
|
|
||||||
|
|
||||||
// 上一条件不满足时，最后尝试放入不忙的池子
|
|
||||||
return newShardId == null? this.selectAndEmptySuitableThreadPool(shardIdPhysicalClusterIdListMap, notBusyShardIdList): newShardId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Long selectAndEmptySuitableThreadPool(Map<Long, List<Long>> shardIdPhysicalClusterIdListMap, List<Long> taskThreadPoolList) {
|
|
||||||
if (taskThreadPoolList.size() < 2) {
|
|
||||||
// 没有空闲的线程池队列
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 将两个非忙的合并，空出一个新的交给需要的
|
|
||||||
Long firstNotBusyShardId = taskThreadPoolList.remove((int) 0);
|
|
||||||
Long secondNotBusyShardId = taskThreadPoolList.remove((int) 0);
|
|
||||||
|
|
||||||
List<Long> physicalClusterIdList = shardIdPhysicalClusterIdListMap.get(secondNotBusyShardId);
|
|
||||||
if (physicalClusterIdList == null || physicalClusterIdList.isEmpty()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (Long physicalClusterId: physicalClusterIdList) {
|
|
||||||
modifyPhysicalClusterIdAndShardIdCache(physicalClusterId, firstNotBusyShardId);
|
|
||||||
}
|
|
||||||
|
|
||||||
return secondNotBusyShardId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private synchronized Long modifyPhysicalClusterIdAndShardIdCache(Long physicalClusterId, Long shardId) {
|
|
||||||
if (shardId == null) {
|
|
||||||
shardId = shardIdx.incrementAndGet() % futureUtilNum;
|
|
||||||
}
|
|
||||||
|
|
||||||
PHYSICAL_CLUSTER_ID_SHARD_ID_CACHE.put(physicalClusterId, shardId);
|
|
||||||
return shardId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private synchronized FutureWaitUtil<Void> closeOldAndCreateNew(Long shardId) {
|
|
||||||
// 新的
|
|
||||||
FutureWaitUtil<Void> newFutureUtil = FutureWaitUtil.init(
|
|
||||||
"MetricCollect-Shard-" + shardId,
|
|
||||||
this.futureUtilThreadNum,
|
|
||||||
this.futureUtilThreadNum,
|
|
||||||
this.futureUtilQueueSize
|
|
||||||
);
|
|
||||||
|
|
||||||
// 存储新的，返回旧的
|
|
||||||
FutureWaitUtil<Void> oldFutureUtil = SHARD_ID_FUTURE_UTIL_MAP.put(shardId, newFutureUtil);
|
|
||||||
|
|
||||||
// 为空，则直接返回
|
|
||||||
if (oldFutureUtil == null) {
|
|
||||||
return newFutureUtil;
|
|
||||||
}
|
|
||||||
|
|
||||||
LOGGER.error("close old ThreadPoolExecutor and create new, shardId:{}.", shardId);
|
|
||||||
try {
|
|
||||||
oldFutureUtil.shutdownNow();
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.error("close old ThreadPoolExecutor and create new, shutdownNow failed, shardId:{}.", shardId, e);
|
|
||||||
}
|
|
||||||
|
|
||||||
return newFutureUtil;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.BaseESPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.BaseMetricESDAO;
|
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
public abstract class AbstractMetricESSender {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(AbstractMetricESSender.class);
|
|
||||||
|
|
||||||
private static final int THRESHOLD = 100;
|
|
||||||
|
|
||||||
private static final FutureUtil<Void> esExecutor = FutureUtil.init(
|
|
||||||
"MetricsESSender",
|
|
||||||
10,
|
|
||||||
20,
|
|
||||||
10000
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 根据不同监控维度来发送
|
|
||||||
*/
|
|
||||||
protected boolean send2es(String index, List<? extends BaseESPO> statsList) {
|
|
||||||
LOGGER.info("method=send2es||indexName={}||metricsSize={}||msg=send metrics to es", index, statsList.size());
|
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(statsList)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
BaseMetricESDAO baseMetricESDao = BaseMetricESDAO.getByStatsType(index);
|
|
||||||
if (Objects.isNull(baseMetricESDao)) {
|
|
||||||
LOGGER.error("method=send2es||indexName={}||errMsg=find dao failed", index);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < statsList.size(); i += THRESHOLD) {
|
|
||||||
final int idxStart = i;
|
|
||||||
|
|
||||||
// 异步发送
|
|
||||||
esExecutor.submitTask(
|
|
||||||
() -> baseMetricESDao.batchInsertStats(statsList.subList(idxStart, Math.min(idxStart + THRESHOLD, statsList.size())))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.connect;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectClusterMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectClusterMetricPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CONNECT_CLUSTER_INDEX;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/11/7
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class ConnectClusterMetricESSender extends AbstractMetricESSender implements ApplicationListener<ConnectClusterMetricEvent> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(ConnectClusterMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("class=ConnectClusterMetricESSender||method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(ConnectClusterMetricEvent event) {
|
|
||||||
send2es(CONNECT_CLUSTER_INDEX, ConvertUtil.list2List(event.getConnectClusterMetrics(), ConnectClusterMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.connect;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectorMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectorMetricPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CONNECT_CONNECTOR_INDEX;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author wyb
|
|
||||||
* @date 2022/11/7
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class ConnectorMetricESSender extends AbstractMetricESSender implements ApplicationListener<ConnectorMetricEvent> {
|
|
||||||
protected static final ILog LOGGER = LogFactory.getLog(ConnectorMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("class=ConnectorMetricESSender||method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(ConnectorMetricEvent event) {
|
|
||||||
send2es(CONNECT_CONNECTOR_INDEX, ConvertUtil.list2List(event.getConnectorMetricsList(), ConnectorMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BrokerMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.BrokerMetricPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.BROKER_INDEX;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class BrokerMetricESSender extends AbstractMetricESSender implements ApplicationListener<BrokerMetricEvent> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(BrokerMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(BrokerMetricEvent event) {
|
|
||||||
send2es(BROKER_INDEX, ConvertUtil.list2List(event.getBrokerMetrics(), BrokerMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.ClusterMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ClusterMetricPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CLUSTER_INDEX;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class ClusterMetricESSender extends AbstractMetricESSender implements ApplicationListener<ClusterMetricEvent> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(ClusterMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(ClusterMetricEvent event) {
|
|
||||||
send2es(CLUSTER_INDEX, ConvertUtil.list2List(event.getClusterMetrics(), ClusterMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.GroupMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.GroupMetricPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.GROUP_INDEX;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class GroupMetricESSender extends AbstractMetricESSender implements ApplicationListener<GroupMetricEvent> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(GroupMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(GroupMetricEvent event) {
|
|
||||||
send2es(GROUP_INDEX, ConvertUtil.list2List(event.getGroupMetrics(), GroupMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.PartitionMetricEvent;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.PartitionMetricPO;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.PARTITION_INDEX;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class PartitionMetricESSender extends AbstractMetricESSender implements ApplicationListener<PartitionMetricEvent> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(PartitionMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(PartitionMetricEvent event) {
|
|
||||||
send2es(PARTITION_INDEX, ConvertUtil.list2List(event.getPartitionMetrics(), PartitionMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.collector.sink.kafka;
|
|
||||||
|
|
||||||
import com.didiglobal.logi.log.ILog;
|
|
||||||
import com.didiglobal.logi.log.LogFactory;
|
|
||||||
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.*;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.*;
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
|
||||||
import org.springframework.context.ApplicationListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
|
||||||
|
|
||||||
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.TOPIC_INDEX;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class TopicMetricESSender extends AbstractMetricESSender implements ApplicationListener<TopicMetricEvent> {
|
|
||||||
private static final ILog LOGGER = LogFactory.getLog(TopicMetricESSender.class);
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init(){
|
|
||||||
LOGGER.info("method=init||msg=init finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onApplicationEvent(TopicMetricEvent event) {
|
|
||||||
send2es(TOPIC_INDEX, ConvertUtil.list2List(event.getTopicMetrics(), TopicMetricPO.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user