From 61efdf492f23b20913e4b9fa1ea6ef8327cca800 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Tue, 13 Sep 2022 23:20:41 +0800 Subject: [PATCH 01/18] =?UTF-8?q?=E6=B7=BB=E5=8A=A0docker-compose=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 95 +++ km-dist/docker/knowstreaming-es/Dockerfile | 6 + .../docker/knowstreaming-manage/Dockerfile | 10 + .../docker/knowstreaming-manage/esinit/es.sh | 2 + .../esinit/ks_kafka_broker_metric | 102 +++ .../esinit/ks_kafka_cluster_metric | 187 +++++ .../esinit/ks_kafka_group_metric | 75 ++ .../esinit/ks_kafka_partition_metric | 66 ++ .../esinit/ks_kafka_replication_metric | 23 + .../esinit/ks_kafka_topic_metric | 117 ++++ .../knowstreaming-manage/esinit/template.sh | 662 ++++++++++++++++++ km-dist/docker/knowstreaming-mysql/Dockerfile | 7 + km-dist/docker/knowstreaming-mysql/mysql.cnf | 15 + .../knowstreaming-mysql/sql/ddl-ks-km.sql | 356 ++++++++++ .../knowstreaming-mysql/sql/ddl-logi-job.sql | 119 ++++ .../sql/ddl-logi-security.sql | 205 ++++++ .../knowstreaming-mysql/sql/dml-ks-km.sql | 7 + .../knowstreaming-mysql/sql/dml-logi.sql | 98 +++ km-dist/docker/knowstreaming-web/Dockerfile | 6 + .../knowstreaming-web/conf/knowStreaming.conf | 46 ++ km-dist/init/sql/ddl-logi-security.sql | 2 +- 21 files changed, 2205 insertions(+), 1 deletion(-) create mode 100644 km-dist/docker/docker-compose.yml create mode 100644 km-dist/docker/knowstreaming-es/Dockerfile create mode 100644 km-dist/docker/knowstreaming-manage/Dockerfile create mode 100644 km-dist/docker/knowstreaming-manage/esinit/es.sh create mode 100644 km-dist/docker/knowstreaming-manage/esinit/ks_kafka_broker_metric create mode 100644 km-dist/docker/knowstreaming-manage/esinit/ks_kafka_cluster_metric create mode 100644 km-dist/docker/knowstreaming-manage/esinit/ks_kafka_group_metric create mode 100644 km-dist/docker/knowstreaming-manage/esinit/ks_kafka_partition_metric create mode 100644 km-dist/docker/knowstreaming-manage/esinit/ks_kafka_replication_metric create mode 100644 km-dist/docker/knowstreaming-manage/esinit/ks_kafka_topic_metric create mode 100644 km-dist/docker/knowstreaming-manage/esinit/template.sh create mode 100644 km-dist/docker/knowstreaming-mysql/Dockerfile create mode 100644 km-dist/docker/knowstreaming-mysql/mysql.cnf create mode 100644 km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql create mode 100644 km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql create mode 100644 km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql create mode 100644 km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql create mode 100644 km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql create mode 100644 km-dist/docker/knowstreaming-web/Dockerfile create mode 100644 km-dist/docker/knowstreaming-web/conf/knowStreaming.conf diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml new file mode 100644 index 00000000..ee4b3466 --- /dev/null +++ b/km-dist/docker/docker-compose.yml @@ -0,0 +1,95 @@ +version: "3" + +services: + + knowstreaming-manager: + image: ks_manage:1.0 + container_name: knowstreaming-manager + privileged: true + restart: always + depends_on: + - elasticsearch_single + - kmysql + links: + - elasticsearch_single + - kmysql + expose: + - 80 + #ports: + # - '80:80' + environment: + TZ: Asia/Shanghai + server.port: 80 + spring.datasource.know-streaming.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + spring.datasource.know-streaming.username: root + spring.datasource.know-streaming.password: admin2022_ + + spring.logi-job.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + spring.logi-job.username: root + spring.logi-job.password: admin2022_ + + spring.logi-security.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + spring.logi-security.username: root + spring.logi-security.password: admin2022_ + spring.logi-security.resource-extend-bean-name: myResourceExtendImpl + client-pool.kafka-consumer.max-total-client-num: 20 + + es.client.address: elasticsearch_single:9200 + ES_CLIENT_ADDRESS: elasticsearch_single:9200 + log.path: /logs + + JAVA_OPTS: -Xmx1g -Xms1g -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 + + networks: + - ks + volumes: + - /Users/haoqi/Downloads/ks/manage/log:/logs + knowstreaming-ui: + image: ks_ui:1.0 + container_name: knowstreaming-ui + restart: always + ports: + - '80:80' + environment: + TZ: Asia/Shanghai + depends_on: + - knowstreaming-manager + links: + - knowstreaming-manager + networks: + - ks + + + elasticsearch_single: + image: docker.io/library/elasticsearch:7.6.2 + container_name: elasticsearch_single + ports: + - '9200:9200' + - '9300:9300' + environment: + TZ: Asia/Shanghai + ES_JAVA_OPTS: -Xms512m -Xmx512m + discovery.type: single-node + volumes: + - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data + networks: + - ks + + + kmysql: + image: ks_mysql:1.0 + container_name: kmysql + environment: + TZ: Asia/Shanghai + MYSQL_ROOT_PASSWORD: admin2022_ + MYSQL_DATABASE: know_streaming + networks: + - ks + expose: + - 3306 + volumes: + - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql + +networks: + ks: + driver: bridge \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-es/Dockerfile b/km-dist/docker/knowstreaming-es/Dockerfile new file mode 100644 index 00000000..6a6f6143 --- /dev/null +++ b/km-dist/docker/knowstreaming-es/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.io/library/elasticsearch:7.6.2 +LABEL author=haoqi6677 +#EXPOSE 9200 +#EXPOSE 9300 +#ARG INIT_SOURCE_FILE=./esinit/* +#COPY ${INIT_SOURCE_FILE} / \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile new file mode 100644 index 00000000..021a592d --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -0,0 +1,10 @@ +FROM bitnami/java:11 +ARG JAVA_OPTS +ARG JAR_FILE=./jar/*.jar +ARG INIT_ES_FILE=./esinit/* +COPY ${JAR_FILE} /app/app.jar +COPY ${INIT_ES_FILE} / + +ENTRYPOINT ["/bin/bash","/es.sh"] +CMD ["dumb-init","java","${JAVA_OPTS}", "-jar", "/app/app.jar"] +#ENTRYPOINT java -jar /app/app.jar \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/es.sh b/km-dist/docker/knowstreaming-manage/esinit/es.sh new file mode 100644 index 00000000..9bd811bc --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/es.sh @@ -0,0 +1,2 @@ +/bin/bash /template.sh start +java ${JAVA_OPTS} -jar /app/app.jar \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_broker_metric b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_broker_metric new file mode 100644 index 00000000..78933a4d --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_broker_metric @@ -0,0 +1,102 @@ +PUT _template/ks_kafka_broker_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_broker_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "NetworkProcessorAvgIdle" : { + "type" : "float" + }, + "UnderReplicatedPartitions" : { + "type" : "float" + }, + "BytesIn_min_15" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "RequestHandlerAvgIdle" : { + "type" : "float" + }, + "connectionsCount" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "TotalRequestQueueSize" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "TotalResponseQueueSize" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_cluster_metric b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_cluster_metric new file mode 100644 index 00000000..76d51cbd --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_cluster_metric @@ -0,0 +1,187 @@ +PUT _template/ks_kafka_cluster_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "Connections" : { + "type" : "double" + }, + "BytesIn_min_15" : { + "type" : "double" + }, + "PartitionURP" : { + "type" : "double" + }, + "HealthScore_Topics" : { + "type" : "double" + }, + "EventQueueSize" : { + "type" : "double" + }, + "ActiveControllerCount" : { + "type" : "double" + }, + "GroupDeads" : { + "type" : "double" + }, + "BytesIn_min_5" : { + "type" : "double" + }, + "HealthCheckTotal_Topics" : { + "type" : "double" + }, + "Partitions" : { + "type" : "double" + }, + "BytesOut" : { + "type" : "double" + }, + "Groups" : { + "type" : "double" + }, + "BytesOut_min_15" : { + "type" : "double" + }, + "TotalRequestQueueSize" : { + "type" : "double" + }, + "HealthCheckPassed_Groups" : { + "type" : "double" + }, + "TotalProduceRequests" : { + "type" : "double" + }, + "HealthCheckPassed" : { + "type" : "double" + }, + "TotalLogSize" : { + "type" : "double" + }, + "GroupEmptys" : { + "type" : "double" + }, + "PartitionNoLeader" : { + "type" : "double" + }, + "HealthScore_Brokers" : { + "type" : "double" + }, + "Messages" : { + "type" : "double" + }, + "Topics" : { + "type" : "double" + }, + "PartitionMinISR_E" : { + "type" : "double" + }, + "HealthCheckTotal" : { + "type" : "double" + }, + "Brokers" : { + "type" : "double" + }, + "Replicas" : { + "type" : "double" + }, + "HealthCheckTotal_Groups" : { + "type" : "double" + }, + "GroupRebalances" : { + "type" : "double" + }, + "MessageIn" : { + "type" : "double" + }, + "HealthScore" : { + "type" : "double" + }, + "HealthCheckPassed_Topics" : { + "type" : "double" + }, + "HealthCheckTotal_Brokers" : { + "type" : "double" + }, + "PartitionMinISR_S" : { + "type" : "double" + }, + "BytesIn" : { + "type" : "double" + }, + "BytesOut_min_5" : { + "type" : "double" + }, + "GroupActives" : { + "type" : "double" + }, + "MessagesIn" : { + "type" : "double" + }, + "GroupReBalances" : { + "type" : "double" + }, + "HealthCheckPassed_Brokers" : { + "type" : "double" + }, + "HealthScore_Groups" : { + "type" : "double" + }, + "TotalResponseQueueSize" : { + "type" : "double" + }, + "Zookeepers" : { + "type" : "double" + }, + "LeaderMessages" : { + "type" : "double" + }, + "HealthScore_Cluster" : { + "type" : "double" + }, + "HealthCheckPassed_Cluster" : { + "type" : "double" + }, + "HealthCheckTotal_Cluster" : { + "type" : "double" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "type" : "date" + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_group_metric b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_group_metric new file mode 100644 index 00000000..9fe7008b --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_group_metric @@ -0,0 +1,75 @@ +PUT _template/ks_kafka_group_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_group_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "group" : { + "type" : "keyword" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "HealthScore" : { + "type" : "float" + }, + "Lag" : { + "type" : "float" + }, + "OffsetConsumed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "groupMetric" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_partition_metric b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_partition_metric new file mode 100644 index 00000000..2538eba3 --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_partition_metric @@ -0,0 +1,66 @@ +PUT _template/ks_kafka_partition_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_replication_metric b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_replication_metric new file mode 100644 index 00000000..b7beda54 --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_replication_metric @@ -0,0 +1,23 @@ +PUT _template/ks_kafka_replication_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_replication_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_topic_metric b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_topic_metric new file mode 100644 index 00000000..a2456dc6 --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_topic_metric @@ -0,0 +1,117 @@ +PUT _template/ks_kafka_topic_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_topic_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "topic" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "BytesIn_min_15" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "BytesRejected" : { + "type" : "float" + }, + "PartitionURP" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ReplicationCount" : { + "type" : "float" + }, + "ReplicationBytesOut" : { + "type" : "float" + }, + "ReplicationBytesIn" : { + "type" : "float" + }, + "FailedFetchRequests" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "LogSize" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "FailedProduceRequests" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "brokerAgg" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/template.sh b/km-dist/docker/knowstreaming-manage/esinit/template.sh new file mode 100644 index 00000000..bd69d1d1 --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/esinit/template.sh @@ -0,0 +1,662 @@ +esAddrPort=${ES_CLIENT_ADDRESS} +echo "Wait ElasticSearch Start..." +while true +do + curl -s --connect-timeout 10 -o /dev/null http://${esAddrPort}/_cat/nodes >> /dev/null 2>&1 + if [ "$?" != "0" ];then + sleep 1s + else + echo "ElasticSearch Start Success" + break + fi +done + +curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_broker_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_broker_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "NetworkProcessorAvgIdle" : { + "type" : "float" + }, + "UnderReplicatedPartitions" : { + "type" : "float" + }, + "BytesIn_min_15" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "RequestHandlerAvgIdle" : { + "type" : "float" + }, + "connectionsCount" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "TotalRequestQueueSize" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "TotalResponseQueueSize" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_cluster_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "Connections" : { + "type" : "double" + }, + "BytesIn_min_15" : { + "type" : "double" + }, + "PartitionURP" : { + "type" : "double" + }, + "HealthScore_Topics" : { + "type" : "double" + }, + "EventQueueSize" : { + "type" : "double" + }, + "ActiveControllerCount" : { + "type" : "double" + }, + "GroupDeads" : { + "type" : "double" + }, + "BytesIn_min_5" : { + "type" : "double" + }, + "HealthCheckTotal_Topics" : { + "type" : "double" + }, + "Partitions" : { + "type" : "double" + }, + "BytesOut" : { + "type" : "double" + }, + "Groups" : { + "type" : "double" + }, + "BytesOut_min_15" : { + "type" : "double" + }, + "TotalRequestQueueSize" : { + "type" : "double" + }, + "HealthCheckPassed_Groups" : { + "type" : "double" + }, + "TotalProduceRequests" : { + "type" : "double" + }, + "HealthCheckPassed" : { + "type" : "double" + }, + "TotalLogSize" : { + "type" : "double" + }, + "GroupEmptys" : { + "type" : "double" + }, + "PartitionNoLeader" : { + "type" : "double" + }, + "HealthScore_Brokers" : { + "type" : "double" + }, + "Messages" : { + "type" : "double" + }, + "Topics" : { + "type" : "double" + }, + "PartitionMinISR_E" : { + "type" : "double" + }, + "HealthCheckTotal" : { + "type" : "double" + }, + "Brokers" : { + "type" : "double" + }, + "Replicas" : { + "type" : "double" + }, + "HealthCheckTotal_Groups" : { + "type" : "double" + }, + "GroupRebalances" : { + "type" : "double" + }, + "MessageIn" : { + "type" : "double" + }, + "HealthScore" : { + "type" : "double" + }, + "HealthCheckPassed_Topics" : { + "type" : "double" + }, + "HealthCheckTotal_Brokers" : { + "type" : "double" + }, + "PartitionMinISR_S" : { + "type" : "double" + }, + "BytesIn" : { + "type" : "double" + }, + "BytesOut_min_5" : { + "type" : "double" + }, + "GroupActives" : { + "type" : "double" + }, + "MessagesIn" : { + "type" : "double" + }, + "GroupReBalances" : { + "type" : "double" + }, + "HealthCheckPassed_Brokers" : { + "type" : "double" + }, + "HealthScore_Groups" : { + "type" : "double" + }, + "TotalResponseQueueSize" : { + "type" : "double" + }, + "Zookeepers" : { + "type" : "double" + }, + "LeaderMessages" : { + "type" : "double" + }, + "HealthScore_Cluster" : { + "type" : "double" + }, + "HealthCheckPassed_Cluster" : { + "type" : "double" + }, + "HealthCheckTotal_Cluster" : { + "type" : "double" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "type" : "date" + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_group_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_group_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "group" : { + "type" : "keyword" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "HealthScore" : { + "type" : "float" + }, + "Lag" : { + "type" : "float" + }, + "OffsetConsumed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "groupMetric" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_partition_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_replication_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }[root@10-255-0-23 template]# cat ks_kafka_replication_metric +PUT _template/ks_kafka_replication_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_replication_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_topic_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_topic_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "topic" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "BytesIn_min_15" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "BytesRejected" : { + "type" : "float" + }, + "PartitionURP" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ReplicationCount" : { + "type" : "float" + }, + "ReplicationBytesOut" : { + "type" : "float" + }, + "ReplicationBytesIn" : { + "type" : "float" + }, + "FailedFetchRequests" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "LogSize" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "FailedProduceRequests" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "brokerAgg" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +for i in `seq 6`; +do + logdate=_$(date -d "${i} day ago" +%Y-%m-%d) + curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_broker_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_cluster_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_group_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_partition_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_replication_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_topic_metric${logdate} || \ + exit 2 +done + +echo "ElasticSearch Initialize Success" \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile new file mode 100644 index 00000000..1c5d8b9b --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/Dockerfile @@ -0,0 +1,7 @@ +FROM mysql:5.7.39-oracle +EXPOSE 3306 +ARG INIT_SOURCE_FILE=./sql/*.sql +#RUN mkdir /data/mysql +#RUN mkdir /var/lib/mysql-files +COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/ +COPY ./*.cnf /etc/mysql/conf.d/ \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/mysql.cnf b/km-dist/docker/knowstreaming-mysql/mysql.cnf new file mode 100644 index 00000000..37c97010 --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/mysql.cnf @@ -0,0 +1,15 @@ +[client] +default-character-set=utf8 +[mysql] +default-character-set=utf8 +[mysqld] +skip-host-cache +skip-name-resolve +datadir=/data/mysql +socket=/var/lib/mysql/mysql.sock +secure-file-priv=/var/lib/mysql-files +character-set-server=utf8 +user=mysql +symbolic-links=0 +pid-file=/var/run/mysqld/mysqld.pid +sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql b/km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql new file mode 100644 index 00000000..50696917 --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql @@ -0,0 +1,356 @@ +-- KS-KM自身的SQL，KS-KM依赖 Logi-Job 和 Logi-Security，因此另外两个ddl sql文件也需要执行 + +DROP TABLE IF EXISTS `ks_km_broker`; +CREATE TABLE `ks_km_broker` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '物理集群ID', + `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', + `host` varchar(128) NOT NULL DEFAULT '' COMMENT 'broker主机名', + `port` int(16) NOT NULL DEFAULT '-1' COMMENT 'broker端口', + `jmx_port` int(16) NOT NULL DEFAULT '-1' COMMENT 'Jmx端口', + `start_timestamp` bigint(20) NOT NULL DEFAULT '-1' COMMENT '启动时间', + `rack` varchar(128) NOT NULL DEFAULT '' COMMENT 'Rack信息', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1存活，0未存活', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `endpoint_map` varchar(1024) NOT NULL DEFAULT '' COMMENT '监听信息', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_phy_id_broker_id` (`cluster_phy_id`,`broker_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Broker信息表'; + + + +DROP TABLE IF EXISTS `ks_km_broker_config`; +CREATE TABLE `ks_km_broker_config` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', + `config_name` varchar(192) NOT NULL DEFAULT '' COMMENT '配置名称', + `config_value` text COMMENT '配置值', + `diff_type` int(16) NOT NULL DEFAULT '-1' COMMENT '差异类型', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_broker_name` (`cluster_phy_id`,`broker_id`,`config_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Broker配置信息表'; + + + +DROP TABLE IF EXISTS `ks_km_cluster_balance_job`; +CREATE TABLE `ks_km_cluster_balance_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `brokers` varchar(1024) NOT NULL DEFAULT '' COMMENT '均衡节点', + `topic_black_list` varchar(4096) NOT NULL DEFAULT '' COMMENT 'topic黑名单', + `type` int(16) NOT NULL DEFAULT '0' COMMENT '1:周期均衡，2：立即均衡', + `balance_interval_json` text COMMENT '均衡区间详情', + `metric_calculation_period` int(16) NOT NULL DEFAULT '0' COMMENT '指标计算周期，单位分钟', + `reassignment_json` text COMMENT '迁移脚本', + `parallel_num` int(16) NOT NULL DEFAULT '0' COMMENT '任务并行数', + `execution_strategy` int(16) NOT NULL DEFAULT '0' COMMENT '执行策略， 1：优先最大副本，2：优先最小副本', + `throttle_unit_b` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', + `total_reassign_size` double NOT NULL DEFAULT '0' COMMENT '总迁移大小', + `total_reassign_replica_num` int(16) NOT NULL DEFAULT '0' COMMENT '总迁移副本数', + `move_in_topic_list` varchar(4096) NOT NULL DEFAULT '' COMMENT '移入topic', + `broker_balance_detail` text COMMENT '节点均衡详情', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态 1：进行中，2：准备，3，成功，4：失败，5：取消', + `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', + `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务完成时间', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + `description` text COMMENT '备注', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群均衡任务'; + + + +DROP TABLE IF EXISTS `ks_km_cluster_balance_job_config`; +CREATE TABLE `ks_km_cluster_balance_job_config` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `brokers` varchar(256) NOT NULL DEFAULT '' COMMENT '均衡节点', + `topic_black_list` varchar(4096) NOT NULL DEFAULT '' COMMENT 'topic黑名单', + `task_cron` varchar(64) NOT NULL DEFAULT '' COMMENT '任务周期', + `balance_interval_json` text COMMENT '均衡区间详情', + `metric_calculation_period` int(16) NOT NULL DEFAULT '0' COMMENT '指标计算周期，单位分钟', + `reassignment_json` text COMMENT '迁移脚本', + `parallel_num` int(16) NOT NULL DEFAULT '0' COMMENT '任务并行数', + `execution_strategy` int(16) NOT NULL DEFAULT '0' COMMENT '执行策略， 1：优先最大副本，2：优先最小副本', + `throttle_unit_b` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', + `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态 0：未开启，1：开启', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群均衡任务'; + + + +DROP TABLE IF EXISTS `ks_km_cluster_balance_reassign`; +CREATE TABLE `ks_km_cluster_balance_reassign` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `job_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '父任务ID', + `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', + `original_broker_ids` text COMMENT '源BrokerId列表', + `reassign_broker_ids` text COMMENT '目标BrokerId列表', + `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务完成时间', + `extend_data` text COMMENT '扩展数据', + `status` int(16) NOT NULL DEFAULT '2' COMMENT '任务状态', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群平衡迁移详情'; + + + +DROP TABLE IF EXISTS `ks_km_group_member`; +CREATE TABLE `ks_km_group_member` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `topic_name` varchar(192) NOT NULL DEFAULT '' COMMENT 'Topic名称', + `group_name` varchar(192) NOT NULL DEFAULT '' COMMENT 'Group名称', + `kafka_user` varchar(192) NOT NULL DEFAULT '' COMMENT 'Kafka用户', + `state` varchar(64) NOT NULL DEFAULT '' COMMENT '状态', + `member_count` int(11) NOT NULL DEFAULT '0' COMMENT '成员数', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_topic_group` (`cluster_phy_id`,`topic_name`,`group_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='GroupMember信息表'; + + + +DROP TABLE IF EXISTS `ks_km_health_check_result`; +CREATE TABLE `ks_km_health_check_result` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `dimension` int(11) NOT NULL DEFAULT '0' COMMENT '检查维度(0:未知，1:Cluster，2:Broker，3:Topic，4:Group)', + `config_name` varchar(192) NOT NULL DEFAULT '' COMMENT '配置名', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '物理集群ID', + `res_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', + `passed` int(11) NOT NULL DEFAULT '0' COMMENT '检查通过(0:未通过，1:通过)', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_dimension_config_cluster_res` (`dimension`,`config_name`,`cluster_phy_id`,`res_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='健康检查结果'; + + + +DROP TABLE IF EXISTS `ks_km_job`; +CREATE TABLE `ks_km_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', + `job_name` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务名称', + `job_type` tinyint(10) NOT NULL COMMENT '任务类型', + `job_status` tinyint(10) NOT NULL COMMENT '任务状态', + `job_data` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务的详细信息', + `job_desc` varchar(1024) NOT NULL DEFAULT '' COMMENT '任务描述', + `cluster_id` int(11) NOT NULL COMMENT 'kafka集群id', + `target` varchar(8192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务执行对象', + `running_status` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '任务运行详细状态(json), Success：7 Fail：1 Doing：2', + `creator` varchar(45) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '创建者', + `plan_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '计划执行时间', + `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '实际执行时间', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `index_cluster_id` (`cluster_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Job信息'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_acl`; +CREATE TABLE `ks_km_kafka_acl` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `principal` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Kafka用户', + `operation` int(11) NOT NULL DEFAULT '0' COMMENT '操作', + `permission_type` int(11) NOT NULL DEFAULT '0' COMMENT '权限类型(0:未知，1:任意，2:拒绝，3:允许)', + `host` varchar(192) NOT NULL DEFAULT '127.0.0.1' COMMENT '机器', + `resource_type` int(11) NOT NULL DEFAULT '0' COMMENT '资源类型(0:未知，1:任意，2:Topic，3:Group，4:Cluster，5:事务，6:Token)', + `resource_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', + `pattern_type` int(11) NOT NULL COMMENT '匹配类型(0:未知，1:任意，2:Match，3:Literal，4:prefixed)', + `unique_field` varchar(1024) NOT NULL DEFAULT '' COMMENT '唯一字段，由cluster_phy_id等字段组成', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_uniq_field` (`unique_field`), + KEY `idx_cluster_phy_id_principal_res_name` (`cluster_phy_id`,`principal`,`resource_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='ACL信息'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_change_record`; +CREATE TABLE `ks_km_kafka_change_record` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `res_type` int(11) NOT NULL DEFAULT '-1' COMMENT '资源类型', + `res_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', + `operate_type` int(11) NOT NULL DEFAULT '-1' COMMENT '操作类型', + `operate_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '操作时间', + `unique_field` varchar(1024) NOT NULL DEFAULT '' COMMENT '唯一键字段', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `unique_field` (`unique_field`), + KEY `idx_cluster_phy_id` (`cluster_phy_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Kafka变更记录表'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_controller`; +CREATE TABLE `ks_km_kafka_controller` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群id', + `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', + `broker_host` varchar(256) NOT NULL DEFAULT '' COMMENT '主机名', + `broker_rack` varchar(256) NOT NULL DEFAULT '' COMMENT 'BrokerRack信息', + `timestamp` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'controller变更时间，-1表示未存活', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_broker_timestamp` (`cluster_phy_id`,`broker_id`,`timestamp`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='controller记录表'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_user`; +CREATE TABLE `ks_km_kafka_user` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '名称', + `token` varchar(8192) NOT NULL DEFAULT '' COMMENT '密钥', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_phy_id_name` (`cluster_phy_id`,`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Kafka-User信息表'; + + + +DROP TABLE IF EXISTS `ks_km_partition`; +CREATE TABLE `ks_km_partition` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', + `leader_broker_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区的LeaderBroker，-1表示无Leader', + `in_sync_replicas` varchar(512) NOT NULL DEFAULT '-1' COMMENT 'ISR', + `assign_replicas` varchar(512) NOT NULL DEFAULT '-1' COMMENT 'AR', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_topic_partition` (`cluster_phy_id`,`topic_name`,`partition_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Partition信息表'; + + + +DROP TABLE IF EXISTS `ks_km_physical_cluster`; +CREATE TABLE `ks_km_physical_cluster` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '集群id', + `name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群名称', + `zookeeper` varchar(2048) NOT NULL DEFAULT '' COMMENT 'zk地址', + `bootstrap_servers` varchar(2048) NOT NULL DEFAULT '' COMMENT 'server地址', + `kafka_version` varchar(32) NOT NULL DEFAULT '' COMMENT 'kafka版本', + `client_properties` text COMMENT 'Kafka客户端配置', + `jmx_properties` text COMMENT 'JMX配置', + `description` text COMMENT '备注', + `auth_type` int(11) NOT NULL DEFAULT '0' COMMENT '认证类型，-1未知，0:无认证，', + `run_state` tinyint(4) NOT NULL DEFAULT '1' COMMENT '运行状态, 0表示未监控, 1监控中，有ZK，2:监控中，无ZK', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '接入时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='物理集群信息表'; + + + +DROP TABLE IF EXISTS `ks_km_platform_cluster_config`; +CREATE TABLE `ks_km_platform_cluster_config` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', + `cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `value_group` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项组', + `value_name` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项名字', + `value` text COMMENT '配置项的值', + `description` text COMMENT '备注', + `operator` varchar(16) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '操作者', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_id_group_name` (`cluster_id`,`value_group`,`value_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='KS平台集群配置'; + + + +DROP TABLE IF EXISTS `ks_km_reassign_job`; +CREATE TABLE `ks_km_reassign_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `reassignment_json` text COMMENT '迁移计划', + `throttle_unit_byte` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', + `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务完成时间', + `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', + `description` text COMMENT '备注', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='迁移Job信息'; + + + +DROP TABLE IF EXISTS `ks_km_reassign_sub_job`; +CREATE TABLE `ks_km_reassign_sub_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `job_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '父任务ID', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', + `original_broker_ids` text COMMENT '源BrokerId列表', + `reassign_broker_ids` text COMMENT '目标BrokerId列表', + `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务完成时间', + `extend_data` text COMMENT '扩展数据', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='迁移SubJob信息'; + + + +DROP TABLE IF EXISTS `ks_km_topic`; +CREATE TABLE `ks_km_topic` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `replica_num` int(11) NOT NULL DEFAULT '-1' COMMENT '副本数', + `partition_num` int(11) NOT NULL DEFAULT '-1' COMMENT '分区数', + `broker_ids` varchar(2048) NOT NULL DEFAULT '' COMMENT 'BrokerId列表', + `partition_map` text COMMENT '分区分布信息', + `retention_ms` bigint(20) NOT NULL DEFAULT '-2' COMMENT '保存时间，-2：未知，-1：无限制，>=0对应时间，单位ms', + `type` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'Topic类型，默认0，0:普通，1:Kafka内部', + `description` text COMMENT '备注信息', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间(尽量与Topic实际创建时间一致)', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间(尽量与Topic实际创建时间一致)', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_phy_id_topic_name` (`cluster_phy_id`,`topic_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Topic信息表'; + + +DROP TABLE IF EXISTS `ks_km_app_node`; +CREATE TABLE `ks_km_app_node` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `host_name` varchar(256) NOT NULL DEFAULT '' COMMENT 'host', + `ip` varchar(256) NOT NULL DEFAULT '' COMMENT 'ip', + `beat_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'node 的心跳时间', + `app_name` varchar(128) NOT NULL DEFAULT '' COMMENT 'km 集群的应用名', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + KEY `idx_app_host` (`app_name`,`host_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='km集群部署的node信息'; diff --git a/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql b/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql new file mode 100644 index 00000000..3ba22983 --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql @@ -0,0 +1,119 @@ +-- Logi-Job模块的sql，安装KS-KM需要执行该sql + + +DROP TABLE IF EXISTS `logi_job`; +CREATE TABLE `logi_job` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', + `task_code` varchar(255) NOT NULL DEFAULT '' COMMENT '任务code', + `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', + `try_times` int(10) NOT NULL DEFAULT '0' COMMENT '第几次重试', + `worker_code` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `start_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '开始时间', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `job_code` (`job_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='正在执行的job信息'; + + + +DROP TABLE IF EXISTS `logi_job_log`; +CREATE TABLE `logi_job_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'job taskCode', + `task_code` varchar(255) NOT NULL DEFAULT '' COMMENT '任务code', + `task_name` varchar(255) NOT NULL DEFAULT '' COMMENT '任务名称', + `task_desc` varchar(255) NOT NULL DEFAULT '' COMMENT '任务描述', + `task_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '任务id', + `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', + `try_times` int(10) NOT NULL DEFAULT '0' COMMENT '第几次重试', + `worker_code` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器', + `worker_ip` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器ip', + `start_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '开始时间', + `end_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '结束时间', + `status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '执行结果 1成功 2失败 3取消', + `error` text NOT NULL COMMENT '错误信息', + `result` text NOT NULL COMMENT '执行结果', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `index_job_code` (`job_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='job执行历史日志'; + + + +DROP TABLE IF EXISTS `logi_task`; +CREATE TABLE `logi_task` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', + `task_name` varchar(255) NOT NULL DEFAULT '' COMMENT '名称', + `task_desc` varchar(1000) NOT NULL DEFAULT '' COMMENT '任务描述', + `cron` varchar(100) NOT NULL DEFAULT '' COMMENT 'cron 表达式', + `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', + `params` varchar(1000) NOT NULL DEFAULT '' COMMENT '执行参数 map 形式{key1:value1,key2:value2}', + `retry_times` int(10) NOT NULL DEFAULT '0' COMMENT '允许重试次数', + `last_fire_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '上次执行时间', + `timeout` bigint(20) NOT NULL DEFAULT '0' COMMENT '超时 毫秒', + `status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '1等待 2运行中 3暂停', + `sub_task_codes` varchar(1000) NOT NULL DEFAULT '' COMMENT '子任务code列表,逗号分隔', + `consensual` varchar(200) NOT NULL DEFAULT '' COMMENT '执行策略', + `owner` varchar(200) NOT NULL DEFAULT '' COMMENT '责任人', + `task_worker_str` varchar(3000) NOT NULL DEFAULT '' COMMENT '机器执行信息', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `task_code` (`task_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='任务信息'; + + + +DROP TABLE IF EXISTS `logi_task_lock`; +CREATE TABLE `logi_task_lock` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', + `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `expire_time` bigint(20) NOT NULL DEFAULT '0' COMMENT '过期时间', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='任务锁'; + + + +DROP TABLE IF EXISTS `logi_worker`; +CREATE TABLE `logi_worker` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', + `worker_name` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker名', + `ip` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker的ip', + `cpu` int(11) NOT NULL DEFAULT '0' COMMENT 'cpu数量', + `cpu_used` double NOT NULL DEFAULT '0' COMMENT 'cpu使用率', + `memory` double NOT NULL DEFAULT '0' COMMENT '内存,以M为单位', + `memory_used` double NOT NULL DEFAULT '0' COMMENT '内存使用率', + `jvm_memory` double NOT NULL DEFAULT '0' COMMENT 'jvm堆大小，以M为单位', + `jvm_memory_used` double NOT NULL DEFAULT '0' COMMENT 'jvm堆使用率', + `job_num` int(10) NOT NULL DEFAULT '0' COMMENT '正在执行job数', + `heartbeat` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '心跳时间', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `worker_code` (`worker_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker信息'; + + + +DROP TABLE IF EXISTS `logi_worker_blacklist`; +CREATE TABLE `logi_worker_blacklist` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `worker_code` (`worker_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker黑名单列表'; diff --git a/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql b/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql new file mode 100644 index 00000000..7efa1f3a --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql @@ -0,0 +1,205 @@ +#-----------------------创建表----------------------- +DROP TABLE IF EXISTS `logi_security_dept`; +CREATE TABLE `logi_security_dept` +( + id int auto_increment primary key, + dept_name varchar(10) not null comment '部门名', + parent_id int not null comment '父部门id', + leaf tinyint(1) not null comment '是否叶子部门', + level tinyint not null comment 'parentId为0的层级为1', + description varchar(20) null comment '描述', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '部门信息表'; + +DROP TABLE IF EXISTS `logi_security_message`; +CREATE TABLE `logi_security_message` +( + id int auto_increment primary key, + title varchar(60) not null comment '标题', + content varchar(256) null comment '内容', + read_tag tinyint(1) default 0 null comment '是否已读', + oplog_id int null comment '操作日志id', + user_id int null comment '这条消息属于哪个用户的，用户id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '消息中心'; + +DROP TABLE IF EXISTS `logi_security_oplog`; +CREATE TABLE `logi_security_oplog` +( + id int auto_increment primary key, + operator_ip varchar(20) not null comment '操作者ip', + operator varchar(20) null comment '操作者账号', + operate_page varchar(16) not null default '' comment '操作页面', + operate_type varchar(16) not null comment '操作类型', + target_type varchar(16) not null comment '对象分类', + target varchar(128) not null comment '操作对象', + operation_methods varchar(20) not null default '' comment '操作方式', + detail text null comment '日志详情', + create_time timestamp default CURRENT_TIMESTAMP null, + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '操作日志'; + +DROP TABLE IF EXISTS `logi_security_oplog_extra`; +CREATE TABLE `logi_security_oplog_extra` +( + id int auto_increment primary key, + info varchar(16) null comment '信息', + type tinyint not null comment '哪种信息：1：操作页面;2：操作类型;3：对象分类', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '操作日志信息（操作页面、操作类型、对象分类）'; + +DROP TABLE IF EXISTS `logi_security_permission`; +CREATE TABLE `logi_security_permission` +( + id int auto_increment primary key, + permission_name varchar(40) not null comment '权限名字', + parent_id int not null comment '父权限id', + leaf tinyint(1) not null comment '是否叶子权限点（具体的操作）', + level tinyint not null comment '权限点的层级（parentId为0的层级为1）', + description varchar(64) null comment '权限点描述', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '权限表'; + +DROP TABLE IF EXISTS `logi_security_project`; +CREATE TABLE `logi_security_project` +( + id int auto_increment comment '项目id' primary key, + project_code varchar(128) not null comment '项目编号', + project_name varchar(128) not null comment '项目名', + description varchar(512) default '' not null comment '项目描述', + dept_id int not null comment '部门id', + running tinyint(1) default 1 not null comment '启用 or 停用', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '项目表'; + +DROP TABLE IF EXISTS `logi_security_resource_type`; +CREATE TABLE `logi_security_resource_type` +( + id int auto_increment primary key, + type_name varchar(16) null comment '资源类型名', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '资源类型表'; + +DROP TABLE IF EXISTS `logi_security_role`; +CREATE TABLE `logi_security_role` +( + id int auto_increment primary key, + role_code varchar(128) not null comment '角色编号', + role_name varchar(128) not null comment '名称', + description varchar(128) null comment '角色描述', + last_reviser varchar(30) null comment '最后修改人', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '角色信息'; + +DROP TABLE IF EXISTS `logi_security_role_permission`; +CREATE TABLE `logi_security_role_permission` +( + id int auto_increment primary key, + role_id int not null comment '角色id', + permission_id int not null comment '权限id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '角色权限表（只保留叶子权限与角色关系）'; + +DROP TABLE IF EXISTS `logi_security_user`; +CREATE TABLE `logi_security_user` +( + id int auto_increment primary key, + user_name varchar(64) not null comment '用户账号', + pw varchar(2048) not null comment '用户密码', + salt char(5) default '' not null comment '密码盐', + real_name varchar(128) default '' not null comment '真实姓名', + phone char(11) default '' not null comment 'mobile', + email varchar(30) default '' not null comment 'email', + dept_id int null comment '所属部门id', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + create_time timestamp default CURRENT_TIMESTAMP null comment '注册时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户信息'; + +DROP TABLE IF EXISTS `logi_security_user_project`; +CREATE TABLE `logi_security_user_project` +( + id int auto_increment primary key, + user_id int not null comment '用户id', + project_id int not null comment '项目id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户项目关系表（项目负责人）'; + +DROP TABLE IF EXISTS `logi_security_user_resource`; +CREATE TABLE `logi_security_user_resource` +( + id int auto_increment primary key, + user_id int not null comment '用户id', + project_id int not null comment '资源所属项目id', + resource_type_id int not null comment '资源类别id', + resource_id int not null comment '资源id', + control_level tinyint not null comment '管理级别：1（查看权限）2（管理权限）', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户和资源关系表'; + +DROP TABLE IF EXISTS `logi_security_user_role`; +CREATE TABLE `logi_security_user_role` +( + id int auto_increment primary key, + user_id int not null comment '用户id', + role_id int not null comment '角色id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户角色表'; + +-- ---------------------------- +-- Table structure for logi_config +-- ---------------------------- +DROP TABLE IF EXISTS `logi_security_config`; +CREATE TABLE `logi_security_config` +( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', + `value_group` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项组', + `value_name` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项名字', + `value` text COMMENT '配置项的值', + `edit` int(4) NOT NULL DEFAULT '1' COMMENT '是否可以编辑 1 不可编辑（程序获取） 2 可编辑', + `status` int(4) NOT NULL DEFAULT '1' COMMENT '1 正常 2 禁用', + `memo` varchar(1000) NOT NULL DEFAULT '' COMMENT '备注', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` tinyint(1) NOT NULL DEFAULT '0' COMMENT '逻辑删除', + `app_name` varchar(16) COLLATE utf8_bin DEFAULT NULL COMMENT '应用名称', + `operator` varchar(16) COLLATE utf8_bin DEFAULT NULL COMMENT '操作者', + PRIMARY KEY (`id`), + KEY `idx_group_name` (`value_group`,`value_name`) +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 COMMENT='logi配置项'; \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql b/km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql new file mode 100644 index 00000000..2d354a87 --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql @@ -0,0 +1,7 @@ +-- 检查检查配置 +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_CLUSTER_NO_CONTROLLER','{ \"value\": 1, \"weight\": 30 } ','集群Controller数正常','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_BROKER_REQUEST_QUEUE_FULL','{ \"value\": 10, \"weight\": 20 } ','Broker-RequestQueueSize指标','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_BROKER_NETWORK_PROCESSOR_AVG_IDLE_TOO_LOW','{ \"value\": 0.8, \"weight\": 20 } ','Broker-NetworkProcessorAvgIdlePercent指标','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_GROUP_RE_BALANCE_TOO_FREQUENTLY','{\n \"latestMinutes\": 10,\n \"detectedTimes\": 8,\n \"weight\": 10\n}\n','Group的re-balance频率','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_TOPIC_NO_LEADER','{ \"value\": 1, \"weight\": 10 } ','Topic 无Leader数','know-stream'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_TOPIC_UNDER_REPLICA_TOO_LONG','{ \"latestMinutes\": 10, \"detectedTimes\": 8, \"weight\": 10 } ','Topic 未同步持续时间','know-streaming'); diff --git a/km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql b/km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql new file mode 100644 index 00000000..927a0bcc --- /dev/null +++ b/km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql @@ -0,0 +1,98 @@ +-- 初始化权限 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1593', '多集群管理', '0', '0', '1', '多集群管理', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1595', '系统管理', '0', '0', '1', '系统管理', '0', 'know-streaming'); + +-- 多集群管理权限 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1597', '接入集群', '1593', '1', '2', '接入集群', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1599', '删除集群', '1593', '1', '2', '删除集群', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1601', 'Cluster-修改集群信息', '1593', '1', '2', 'Cluster-修改集群信息', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1603', 'Cluster-修改健康规则', '1593', '1', '2', 'Cluster-修改健康规则', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1605', 'Broker-修改Broker配置', '1593', '1', '2', 'Broker-修改Broker配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1607', 'Topic-新增Topic', '1593', '1', '2', 'Topic-新增Topic', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1609', 'Topic-扩分区', '1593', '1', '2', 'Topic-扩分区', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1611', 'Topic-删除Topic', '1593', '1', '2', 'Topic-扩分区', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1613', 'Topic-重置Offset', '1593', '1', '2', 'Topic-重置Offset', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1615', 'Topic-修改Topic配置', '1593', '1', '2', 'Topic-修改Topic配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1617', 'Consumers-重置Offset', '1593', '1', '2', 'Consumers-重置Offset', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1619', 'Test-Producer', '1593', '1', '2', 'Test-Producer', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1621', 'Test-Consumer', '1593', '1', '2', 'Test-Consumer', '0', 'know-streaming'); + +-- 系统管理权限 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1623', '配置管理-新增配置', '1595', '1', '2', '配置管理-新增配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1625', '配置管理-编辑配置', '1595', '1', '2', '配置管理-编辑配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1627', '配置管理-删除配置', '1595', '1', '2', '配置管理-删除配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1629', '用户管理-新增人员', '1595', '1', '2', '用户管理-新增人员', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1631', '用户管理-编辑人员', '1595', '1', '2', '用户管理-编辑人员', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1633', '用户管理-修改人员密码', '1595', '1', '2', '用户管理-修改人员密码', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1635', '用户管理-删除人员', '1595', '1', '2', '用户管理-删除人员', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1637', '用户管理-新增角色', '1595', '1', '2', '用户管理-新增角色', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1639', '用户管理-编辑角色', '1595', '1', '2', '用户管理-编辑角色', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1641', '用户管理-分配用户角色', '1595', '1', '2', '用户管理-分配用户角色', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1643', '用户管理-删除角色', '1595', '1', '2', '用户管理-删除角色', '0', 'know-streaming'); + +-- 多集群管理权限2022-09-06新增 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2000', '多集群管理查看', '1593', '1', '2', '多集群管理查看', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2002', 'Topic-迁移副本', '1593', '1', '2', 'Topic-迁移副本', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2004', 'Topic-扩缩副本', '1593', '1', '2', 'Topic-扩缩副本', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2006', 'Cluster-LoadReBalance-周期均衡', '1593', '1', '2', 'Cluster-LoadReBalance-周期均衡', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2008', 'Cluster-LoadReBalance-立即均衡', '1593', '1', '2', 'Cluster-LoadReBalance-立即均衡', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2010', 'Cluster-LoadReBalance-设置集群规格', '1593', '1', '2', 'Cluster-LoadReBalance-设置集群规格', '0', 'know-streaming'); + + +-- 系统管理权限2022-09-06新增 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('3000', '系统管理查看', '1595', '1', '2', '系统管理查看', '0', 'know-streaming'); + + + + + + +-- 初始化用户 +INSERT INTO `logi_security_user` (`id`, `user_name`, `pw`, `real_name`, `is_delete`, `app_name`) VALUES ('1', 'admin', 'V1ZkU2RHRlhOSGxOUkVsNVdETjBRVlp0Y0V0T1IwWnlaVEZ6YWxGRVJrRkpNVEU1VTJwYVUySkhlRzlSU0RBOWUwQldha28wWVd0N1d5TkFNa0FqWFgxS05sSnNiR2hBZlE9PXtAVmpKNGFre1sjQDNAI119SjZSbGxoQH0=Mv{#cdRgJ45Lqx}3IubEW87!==', '系统管理员', '0', 'know-streaming'); + +-- 初始化角色 +INSERT INTO `logi_security_role` (`id`, `role_code`, `role_name`, `description`, `last_reviser`, `is_delete`, `app_name`) VALUES ('1677', 'r15477137', '管理员角色', '包含系统所有权限', 'admin', '0', 'know-streaming'); + +-- 初始化角色权限关系 +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1597', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1599', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1601', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1603', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1605', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1607', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1609', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1611', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1613', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1615', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1617', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1619', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1621', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1593', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1623', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1625', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1627', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1629', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1631', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1633', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1635', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1637', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1639', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1641', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1643', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1595', '0', 'know-streaming'); + +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2000', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2002', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2004', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2006', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2008', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2010', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '3000', '0', 'know-streaming'); + +-- 初始化 用户角色关系 +INSERT INTO `logi_security_user_role` (`id`, `user_id`, `role_id`, `is_delete`, `app_name`) VALUES ('1', '1', '1677', '0', 'know-streaming'); + +INSERT INTO `logi_security_config` +(`value_group`,`value_name`,`value`,`edit`,`status`,`memo`,`is_delete`,`app_name`,`operator`) +VALUES +('SECURITY.LOGIN','SECURITY.TRICK_USERS','[\n \"admin\"\n]',1,1,'允许跳过登录的用户',0,'know-streaming','admin'); \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-web/Dockerfile b/km-dist/docker/knowstreaming-web/Dockerfile new file mode 100644 index 00000000..f1bb3a19 --- /dev/null +++ b/km-dist/docker/knowstreaming-web/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.io/library/nginx:1.23.1 +WORKDIR /pub +EXPOSE 80 +COPY ./conf/*.conf /etc/nginx/conf.d/default.conf +COPY ./pub /pub +# HEALTHCHECK --interval=30s --timeout=30s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-web/conf/knowStreaming.conf b/km-dist/docker/knowstreaming-web/conf/knowStreaming.conf new file mode 100644 index 00000000..9f5007ff --- /dev/null +++ b/km-dist/docker/knowstreaming-web/conf/knowStreaming.conf @@ -0,0 +1,46 @@ +server { + listen 80; + server_name localhost; + gzip on; + gzip_buffers 16 8k; + gzip_comp_level 4; + gzip_http_version 1.0; + gzip_min_length 1280; + gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; + gzip_vary on; + root /pub; + location / { + root /pub; + if ($request_filename ~* .*\.(?:htm|html|json)$) { + add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; + } + try_files $uri /layout/index.html; + } + location ~* \.(json)$ { + add_header Cache-Control no-cache; + } + location @kmfallback { + } + #location ~ ^/(clusters|config|cluster|login) { + # rewrite ^.*$ /; + #} + location ~ ^/ks-km/api/v3 { + #rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; + proxy_pass http://knowstreaming-manager; + #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; + #proxy_cookie_path /ks-km/ /; + #proxy_set_header Host $host; + #proxy_set_header Referer $http_referer; + #proxy_set_header Cookie $http_cookie; + #proxy_set_header X-Real-Ip $remote_addr; + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~ ^/logi-security/api/v1 { + #rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; + proxy_pass http://knowstreaming-manager; + #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; + } + location ~ ^/(401|403|404|500){ + rewrite ^.*$ /; + } + } \ No newline at end of file diff --git a/km-dist/init/sql/ddl-logi-security.sql b/km-dist/init/sql/ddl-logi-security.sql index 69fcdc66..7efa1f3a 100644 --- a/km-dist/init/sql/ddl-logi-security.sql +++ b/km-dist/init/sql/ddl-logi-security.sql @@ -38,7 +38,7 @@ CREATE TABLE `logi_security_oplog` operate_page varchar(16) not null default '' comment '操作页面', operate_type varchar(16) not null comment '操作类型', target_type varchar(16) not null comment '对象分类', - target varchar(20) not null comment '操作对象', + target varchar(128) not null comment '操作对象', operation_methods varchar(20) not null default '' comment '操作方式', detail text null comment '日志详情', create_time timestamp default CURRENT_TIMESTAMP null, From 0c14c641d00483e77cd7dccb55703be41a73d624 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Thu, 15 Sep 2022 14:26:45 +0800 Subject: [PATCH 02/18] =?UTF-8?q?1.=20=E6=B7=BB=E5=8A=A0docker-compose?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E6=96=B9=E5=BC=8F=202.=20=E6=9B=B4=E6=94=B9m?= =?UTF-8?q?anage=E6=9C=8D=E5=8A=A1=E5=88=9D=E5=A7=8B=E5=8C=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=203.=20=E6=9B=B4=E6=94=B9es=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 40 +++---- km-dist/docker/knowstreaming-es/Dockerfile | 6 - .../docker/knowstreaming-manage/Dockerfile | 9 +- .../knowstreaming-manage/conf/application.yml | 107 ++++++++++++++++++ .../docker/knowstreaming-manage/esinit/es.sh | 9 +- .../knowstreaming-manage/esinit/template.sh | 41 ++++--- km-dist/docker/knowstreaming-mysql/Dockerfile | 2 +- 7 files changed, 157 insertions(+), 57 deletions(-) delete mode 100644 km-dist/docker/knowstreaming-es/Dockerfile create mode 100644 km-dist/docker/knowstreaming-manage/conf/application.yml diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index ee4b3466..3e902498 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -15,31 +15,17 @@ services: - kmysql expose: - 80 - #ports: - # - '80:80' environment: TZ: Asia/Shanghai - server.port: 80 - spring.datasource.know-streaming.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - spring.datasource.know-streaming.username: root - spring.datasource.know-streaming.password: admin2022_ - spring.logi-job.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - spring.logi-job.username: root - spring.logi-job.password: admin2022_ + SERVER_MYSQL_ADDRESS: kmysql:3306 + SERVER_MYSQL_DB: know_streaming + SERVER_MYSQL_USER: root + SERVER_MYSQL_PASSWORD: admin2022_ - spring.logi-security.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - spring.logi-security.username: root - spring.logi-security.password: admin2022_ - spring.logi-security.resource-extend-bean-name: myResourceExtendImpl - client-pool.kafka-consumer.max-total-client-num: 20 - - es.client.address: elasticsearch_single:9200 - ES_CLIENT_ADDRESS: elasticsearch_single:9200 - log.path: /logs + SERVER_ES_ADDRESS: elasticsearch_single:9200 JAVA_OPTS: -Xmx1g -Xms1g -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 - networks: - ks volumes: @@ -63,15 +49,20 @@ services: elasticsearch_single: image: docker.io/library/elasticsearch:7.6.2 container_name: elasticsearch_single - ports: - - '9200:9200' - - '9300:9300' + restart: always + expose: + - 9200 + - 9300 +# 对外访问 +# ports: +# - '9200:9200' +# - '9300:9300' environment: TZ: Asia/Shanghai ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node - volumes: - - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data +# volumes: +# - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data networks: - ks @@ -79,6 +70,7 @@ services: kmysql: image: ks_mysql:1.0 container_name: kmysql + restart: always environment: TZ: Asia/Shanghai MYSQL_ROOT_PASSWORD: admin2022_ diff --git a/km-dist/docker/knowstreaming-es/Dockerfile b/km-dist/docker/knowstreaming-es/Dockerfile deleted file mode 100644 index 6a6f6143..00000000 --- a/km-dist/docker/knowstreaming-es/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM docker.io/library/elasticsearch:7.6.2 -LABEL author=haoqi6677 -#EXPOSE 9200 -#EXPOSE 9300 -#ARG INIT_SOURCE_FILE=./esinit/* -#COPY ${INIT_SOURCE_FILE} / \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index 021a592d..05e28155 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -1,10 +1,11 @@ FROM bitnami/java:11 -ARG JAVA_OPTS -ARG JAR_FILE=./jar/*.jar -ARG INIT_ES_FILE=./esinit/* +ENV JAR_FILE=./jar/*.jar +ENV INIT_ES_FILE=./esinit/* +ENV CONFIG_FILE=./conf/* COPY ${JAR_FILE} /app/app.jar COPY ${INIT_ES_FILE} / +COPY ${CONFIG_FILE} /conf/ ENTRYPOINT ["/bin/bash","/es.sh"] -CMD ["dumb-init","java","${JAVA_OPTS}", "-jar", "/app/app.jar"] +CMD ["/bin/bash","/es.sh"] #ENTRYPOINT java -jar /app/app.jar \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/conf/application.yml b/km-dist/docker/knowstreaming-manage/conf/application.yml new file mode 100644 index 00000000..80323cd0 --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/conf/application.yml @@ -0,0 +1,107 @@ +server: + port: 80 # 服务端口 + tomcat: + accept-count: 1000 + max-connections: 10000 + +spring: + application: + name: know-streaming + profiles: + active: dev + main: + allow-bean-definition-overriding: true + jackson: + time-zone: GMT+8 + datasource: + know-streaming: # know-streaming 自身数据库的配置 + jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + username: SERVER_MYSQL_USER + password: SERVER_MYSQL_PASSWORD + driver-class-name: org.mariadb.jdbc.Driver + maximum-pool-size: 20 + idle-timeout: 30000 + connection-test-query: SELECT 1 + logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 + jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + username: SERVER_MYSQL_USER + password: SERVER_MYSQL_PASSWORD + driver-class-name: org.mariadb.jdbc.Driver + max-lifetime: 60000 + init-sql: true + init-thread-num: 20 + max-thread-num: 50 + log-expire: 3 # 日志保存天数，以天为单位 + app-name: know-streaming + claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual + logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 + jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + username: SERVER_MYSQL_USER + password: SERVER_MYSQL_PASSWORD + driver-class-name: org.mariadb.jdbc.Driver + app-name: know-streaming + resource-extend-bean-name: myResourceExtendImpl + login-extend-bean-name: logiSecurityDefaultLoginExtendImpl + +logging: + config: classpath:logback-spring.xml + +# 线程池大小相关配置 +thread-pool: + scheduled: + thread-num: 2 # @Scheduled任务的线程池大小，默认是一个 + + collector: # 采集模块的配置 + future-util: # 采集模块线程池配置 + num: 3 # 线程池个数 + thread-num: 64 # 每个线程池核心线程数 + queue-size: 10000 # 每个线程池队列大小 + select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改 + suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改 + + task: # 任务模块的配置 + metrics: # metrics采集任务配置 + thread-num: 18 # metrics采集任务线程池核心线程数 + queue-size: 180 # metrics采集任务线程池队列大小 + metadata: # metadata同步任务配置 + thread-num: 27 # metadata同步任务线程池核心线程数 + queue-size: 270 # metadata同步任务线程池队列大小 + common: # 剩余其他任务配置 + thread-num: 15 # 剩余其他任务线程池核心线程数 + queue-size: 150 # 剩余其他任务线程池队列大小 + + +# 客户端池大小相关配置 +client-pool: + kafka-consumer: + min-idle-client-num: 2 # 最小空闲客户端数 + max-idle-client-num: 20 # 最大空闲客户端数 + max-total-client-num: 20 # 最大客户端数 + borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒 + + +# ES客户端配置 +es: + client: + address: SERVER_ES_ADDRESS + client-cnt: 10 + io-thread-cnt: 2 + max-retry-cnt: 5 + +# 普罗米修斯指标导出相关配置 +management: + endpoints: + web: + base-path: /metrics + exposure: + include: '*' + health: + elasticsearch: + enabled: false + metrics: + export: + prometheus: + descriptions: true + enabled: true + tags: + application: know-streaming \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/es.sh b/km-dist/docker/knowstreaming-manage/esinit/es.sh index 9bd811bc..8d5383a9 100644 --- a/km-dist/docker/knowstreaming-manage/esinit/es.sh +++ b/km-dist/docker/knowstreaming-manage/esinit/es.sh @@ -1,2 +1,9 @@ /bin/bash /template.sh start -java ${JAVA_OPTS} -jar /app/app.jar \ No newline at end of file + +sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml +sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml + +java ${JAVA_OPTS} -jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/template.sh b/km-dist/docker/knowstreaming-manage/esinit/template.sh index bd69d1d1..43097279 100644 --- a/km-dist/docker/knowstreaming-manage/esinit/template.sh +++ b/km-dist/docker/knowstreaming-manage/esinit/template.sh @@ -1,17 +1,16 @@ -esAddrPort=${ES_CLIENT_ADDRESS} echo "Wait ElasticSearch Start..." while true do - curl -s --connect-timeout 10 -o /dev/null http://${esAddrPort}/_cat/nodes >> /dev/null 2>&1 + curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes >> /dev/null 2>&1 if [ "$?" != "0" ];then sleep 1s else - echo "ElasticSearch Start Success" + echo "ElasticSearch Start Initialize" break fi done -curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_broker_metric -d '{ +curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_broker_metric*" @@ -113,7 +112,7 @@ curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' - "aliases" : { } }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_cluster_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_cluster_metric*" @@ -298,9 +297,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_group_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_group_metric*" @@ -373,9 +372,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_partition_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_partition_metric*" @@ -439,9 +438,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_replication_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_partition_metric*" @@ -528,9 +527,9 @@ PUT _template/ks_kafka_replication_metric } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_topic_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_topic_metric*" @@ -645,17 +644,17 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -for i in `seq 6`; +for i in {0..6}; do logdate=_$(date -d "${i} day ago" +%Y-%m-%d) - curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_broker_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_cluster_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_group_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_replication_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_topic_metric${logdate} || \ + curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ exit 2 done diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile index 1c5d8b9b..edaf3ad8 100644 --- a/km-dist/docker/knowstreaming-mysql/Dockerfile +++ b/km-dist/docker/knowstreaming-mysql/Dockerfile @@ -1,6 +1,6 @@ FROM mysql:5.7.39-oracle EXPOSE 3306 -ARG INIT_SOURCE_FILE=./sql/*.sql +ENV INIT_SOURCE_FILE=./sql/*.sql #RUN mkdir /data/mysql #RUN mkdir /var/lib/mysql-files COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/ From 792f8d939db6a71bb7991bc81f2f2f33d1843351 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Thu, 15 Sep 2022 15:06:19 +0800 Subject: [PATCH 03/18] =?UTF-8?q?1.=20=E6=9B=B4=E6=94=B9Dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 11 ++++++++--- km-dist/docker/knowstreaming-manage/Dockerfile | 14 +++++++++----- .../{esinit => init}/ks_kafka_broker_metric | 0 .../{esinit => init}/ks_kafka_cluster_metric | 0 .../{esinit => init}/ks_kafka_group_metric | 0 .../{esinit => init}/ks_kafka_partition_metric | 0 .../{esinit => init}/ks_kafka_replication_metric | 0 .../{esinit => init}/ks_kafka_topic_metric | 0 .../{esinit/es.sh => init/start.sh} | 8 +++++++- .../{esinit => init}/template.sh | 0 km-dist/docker/knowstreaming-mysql/Dockerfile | 9 ++++++--- .../knowstreaming-mysql/{ => conf}/mysql.cnf | 0 km-dist/docker/knowstreaming-web/Dockerfile | 14 +++++++++++--- 13 files changed, 41 insertions(+), 15 deletions(-) rename km-dist/docker/knowstreaming-manage/{esinit => init}/ks_kafka_broker_metric (100%) rename km-dist/docker/knowstreaming-manage/{esinit => init}/ks_kafka_cluster_metric (100%) rename km-dist/docker/knowstreaming-manage/{esinit => init}/ks_kafka_group_metric (100%) rename km-dist/docker/knowstreaming-manage/{esinit => init}/ks_kafka_partition_metric (100%) rename km-dist/docker/knowstreaming-manage/{esinit => init}/ks_kafka_replication_metric (100%) rename km-dist/docker/knowstreaming-manage/{esinit => init}/ks_kafka_topic_metric (100%) rename km-dist/docker/knowstreaming-manage/{esinit/es.sh => init/start.sh} (56%) rename km-dist/docker/knowstreaming-manage/{esinit => init}/template.sh (100%) rename km-dist/docker/knowstreaming-mysql/{ => conf}/mysql.cnf (100%) diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index 3e902498..3ea2655d 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -25,11 +25,12 @@ services: SERVER_ES_ADDRESS: elasticsearch_single:9200 - JAVA_OPTS: -Xmx1g -Xms1g -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 + JAVA_OPTS: -Xmx1g -Xms1g networks: - ks - volumes: - - /Users/haoqi/Downloads/ks/manage/log:/logs +# 日志文件持久化路径 +# volumes: +# - /Users/haoqi/Downloads/ks/manage/log:/logs knowstreaming-ui: image: ks_ui:1.0 container_name: knowstreaming-ui @@ -61,6 +62,8 @@ services: TZ: Asia/Shanghai ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node +# 数据文件持久化路径 +# 记录实时指标信息 # volumes: # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data networks: @@ -79,6 +82,8 @@ services: - ks expose: - 3306 +# 数据文件持久化路径 +# 记录添加的集群基本信息 volumes: - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index 05e28155..e22f25d4 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -1,11 +1,15 @@ -FROM bitnami/java:11 +FROM amd64/openjdk:17.0.2 + +LABEL org.opencontainers.image.authors="haoqi123" + +EXPOSE 80 + ENV JAR_FILE=./jar/*.jar -ENV INIT_ES_FILE=./esinit/* +ENV INIT_ES_FILE=./init/* ENV CONFIG_FILE=./conf/* COPY ${JAR_FILE} /app/app.jar COPY ${INIT_ES_FILE} / COPY ${CONFIG_FILE} /conf/ -ENTRYPOINT ["/bin/bash","/es.sh"] -CMD ["/bin/bash","/es.sh"] -#ENTRYPOINT java -jar /app/app.jar \ No newline at end of file +ENTRYPOINT ["/bin/bash","/start.sh"] +CMD ["/bin/bash","/start.sh"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_broker_metric b/km-dist/docker/knowstreaming-manage/init/ks_kafka_broker_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/ks_kafka_broker_metric rename to km-dist/docker/knowstreaming-manage/init/ks_kafka_broker_metric diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_cluster_metric b/km-dist/docker/knowstreaming-manage/init/ks_kafka_cluster_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/ks_kafka_cluster_metric rename to km-dist/docker/knowstreaming-manage/init/ks_kafka_cluster_metric diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_group_metric b/km-dist/docker/knowstreaming-manage/init/ks_kafka_group_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/ks_kafka_group_metric rename to km-dist/docker/knowstreaming-manage/init/ks_kafka_group_metric diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_partition_metric b/km-dist/docker/knowstreaming-manage/init/ks_kafka_partition_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/ks_kafka_partition_metric rename to km-dist/docker/knowstreaming-manage/init/ks_kafka_partition_metric diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_replication_metric b/km-dist/docker/knowstreaming-manage/init/ks_kafka_replication_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/ks_kafka_replication_metric rename to km-dist/docker/knowstreaming-manage/init/ks_kafka_replication_metric diff --git a/km-dist/docker/knowstreaming-manage/esinit/ks_kafka_topic_metric b/km-dist/docker/knowstreaming-manage/init/ks_kafka_topic_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/ks_kafka_topic_metric rename to km-dist/docker/knowstreaming-manage/init/ks_kafka_topic_metric diff --git a/km-dist/docker/knowstreaming-manage/esinit/es.sh b/km-dist/docker/knowstreaming-manage/init/start.sh similarity index 56% rename from km-dist/docker/knowstreaming-manage/esinit/es.sh rename to km-dist/docker/knowstreaming-manage/init/start.sh index 8d5383a9..db322fab 100644 --- a/km-dist/docker/knowstreaming-manage/esinit/es.sh +++ b/km-dist/docker/knowstreaming-manage/init/start.sh @@ -6,4 +6,10 @@ sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml -java ${JAVA_OPTS} -jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file +java -server ${JAVA_OPTS} \ +-XX:+UseStringDeduplication \ +-Dfile.encoding=UTF-8 \ +-Djava.security.egd=file:/dev/./urandom \ +-Duser.timezone=GMT+08 \ +-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log \ +-jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/template.sh b/km-dist/docker/knowstreaming-manage/init/template.sh similarity index 100% rename from km-dist/docker/knowstreaming-manage/esinit/template.sh rename to km-dist/docker/knowstreaming-manage/init/template.sh diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile index edaf3ad8..4e6917c9 100644 --- a/km-dist/docker/knowstreaming-mysql/Dockerfile +++ b/km-dist/docker/knowstreaming-mysql/Dockerfile @@ -1,7 +1,10 @@ FROM mysql:5.7.39-oracle + +LABEL org.opencontainers.image.authors="haoqi123" + EXPOSE 3306 + ENV INIT_SOURCE_FILE=./sql/*.sql -#RUN mkdir /data/mysql -#RUN mkdir /var/lib/mysql-files +ENV MYSQL_CNF_FILE=./conf/*.cnf COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/ -COPY ./*.cnf /etc/mysql/conf.d/ \ No newline at end of file +COPY ${MYSQL_CNF_FILE} /etc/mysql/conf.d/ \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/mysql.cnf b/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf similarity index 100% rename from km-dist/docker/knowstreaming-mysql/mysql.cnf rename to km-dist/docker/knowstreaming-mysql/conf/mysql.cnf diff --git a/km-dist/docker/knowstreaming-web/Dockerfile b/km-dist/docker/knowstreaming-web/Dockerfile index f1bb3a19..4ec52a71 100644 --- a/km-dist/docker/knowstreaming-web/Dockerfile +++ b/km-dist/docker/knowstreaming-web/Dockerfile @@ -1,6 +1,14 @@ FROM docker.io/library/nginx:1.23.1 + +LABEL org.opencontainers.image.authors="haoqi123" + WORKDIR /pub EXPOSE 80 -COPY ./conf/*.conf /etc/nginx/conf.d/default.conf -COPY ./pub /pub -# HEALTHCHECK --interval=30s --timeout=30s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 \ No newline at end of file + +ENV HTML_FILE=./pub +ENV NGINX_CONF=./conf/*.conf + +COPY ${NGINX_CONF} /etc/nginx/conf.d/default.conf +COPY ${HTML_FILE} /pub + +HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 \ No newline at end of file From 27d1b92690be3762466ae9bbbc7d64c4c8d45c7f Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Thu, 15 Sep 2022 16:22:51 +0800 Subject: [PATCH 04/18] =?UTF-8?q?1.=20=E6=B7=BB=E5=8A=A0init=E5=AE=B9?= =?UTF-8?q?=E5=99=A8=EF=BC=8C=E5=8F=AA=E7=94=A8=E4=BA=8E=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96es=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 17 ++++++++++++++--- km-dist/docker/knowstreaming-init/Dockerfile | 9 +++++++++ .../init/ks_kafka_broker_metric | 0 .../init/ks_kafka_cluster_metric | 0 .../init/ks_kafka_group_metric | 0 .../init/ks_kafka_partition_metric | 0 .../init/ks_kafka_replication_metric | 0 .../init/ks_kafka_topic_metric | 0 .../init/template.sh | 0 km-dist/docker/knowstreaming-manage/Dockerfile | 7 +++---- .../docker/knowstreaming-manage/init/start.sh | 2 -- 11 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 km-dist/docker/knowstreaming-init/Dockerfile rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/ks_kafka_broker_metric (100%) rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/ks_kafka_cluster_metric (100%) rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/ks_kafka_group_metric (100%) rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/ks_kafka_partition_metric (100%) rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/ks_kafka_replication_metric (100%) rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/ks_kafka_topic_metric (100%) rename km-dist/docker/{knowstreaming-manage => knowstreaming-init}/init/template.sh (100%) diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index 3ea2655d..446129ec 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -68,7 +68,18 @@ services: # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data networks: - ks - + esinit: + image: esinit:1.0 + container_name: esinit + depends_on: + - elasticsearch_single + links: + - elasticsearch_single + environment: + TZ: Asia/Shanghai + SERVER_ES_ADDRESS: elasticsearch_single:9200 + networks: + - ks kmysql: image: ks_mysql:1.0 @@ -84,8 +95,8 @@ services: - 3306 # 数据文件持久化路径 # 记录添加的集群基本信息 - volumes: - - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql +# volumes: +# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql networks: ks: diff --git a/km-dist/docker/knowstreaming-init/Dockerfile b/km-dist/docker/knowstreaming-init/Dockerfile new file mode 100644 index 00000000..8f47558c --- /dev/null +++ b/km-dist/docker/knowstreaming-init/Dockerfile @@ -0,0 +1,9 @@ +FROM debian:stable-20220912-slim +LABEL org.opencontainers.image.authors="haoqi123" +ENV INIT_ES_FILE=./init/* +COPY ${INIT_ES_FILE} / + +RUN apt-get update \ + && apt-get install -y curl + +ENTRYPOINT ["/bin/bash","/template.sh","start"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/init/ks_kafka_broker_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_broker_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/ks_kafka_broker_metric rename to km-dist/docker/knowstreaming-init/init/ks_kafka_broker_metric diff --git a/km-dist/docker/knowstreaming-manage/init/ks_kafka_cluster_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_cluster_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/ks_kafka_cluster_metric rename to km-dist/docker/knowstreaming-init/init/ks_kafka_cluster_metric diff --git a/km-dist/docker/knowstreaming-manage/init/ks_kafka_group_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_group_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/ks_kafka_group_metric rename to km-dist/docker/knowstreaming-init/init/ks_kafka_group_metric diff --git a/km-dist/docker/knowstreaming-manage/init/ks_kafka_partition_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_partition_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/ks_kafka_partition_metric rename to km-dist/docker/knowstreaming-init/init/ks_kafka_partition_metric diff --git a/km-dist/docker/knowstreaming-manage/init/ks_kafka_replication_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_replication_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/ks_kafka_replication_metric rename to km-dist/docker/knowstreaming-init/init/ks_kafka_replication_metric diff --git a/km-dist/docker/knowstreaming-manage/init/ks_kafka_topic_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_topic_metric similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/ks_kafka_topic_metric rename to km-dist/docker/knowstreaming-init/init/ks_kafka_topic_metric diff --git a/km-dist/docker/knowstreaming-manage/init/template.sh b/km-dist/docker/knowstreaming-init/init/template.sh similarity index 100% rename from km-dist/docker/knowstreaming-manage/init/template.sh rename to km-dist/docker/knowstreaming-init/init/template.sh diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index e22f25d4..c73fd07d 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -5,11 +5,10 @@ LABEL org.opencontainers.image.authors="haoqi123" EXPOSE 80 ENV JAR_FILE=./jar/*.jar -ENV INIT_ES_FILE=./init/* +ENV INIT_FILE=./init/* ENV CONFIG_FILE=./conf/* COPY ${JAR_FILE} /app/app.jar -COPY ${INIT_ES_FILE} / +COPY ${INIT_FILE} / COPY ${CONFIG_FILE} /conf/ -ENTRYPOINT ["/bin/bash","/start.sh"] -CMD ["/bin/bash","/start.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash","/start.sh"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/init/start.sh b/km-dist/docker/knowstreaming-manage/init/start.sh index db322fab..9741d7cf 100644 --- a/km-dist/docker/knowstreaming-manage/init/start.sh +++ b/km-dist/docker/knowstreaming-manage/init/start.sh @@ -1,5 +1,3 @@ -/bin/bash /template.sh start - sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml From 0d9a6702c13e44f37cc9c6dfc0a3ee0e8c74adeb Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Thu, 15 Sep 2022 17:13:58 +0800 Subject: [PATCH 05/18] =?UTF-8?q?1.=20=E6=9B=B4=E6=94=B9es=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96=E8=84=9A=E6=9C=AC=E8=BE=93=E5=87=BA=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E4=B8=BA=E9=87=8D=E5=AE=9A=E5=90=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../init/ks_kafka_broker_metric | 102 ----- .../init/ks_kafka_cluster_metric | 187 --------- .../init/ks_kafka_group_metric | 75 ---- .../init/ks_kafka_partition_metric | 66 ---- .../init/ks_kafka_replication_metric | 23 -- .../init/ks_kafka_topic_metric | 117 ------ .../knowstreaming-init/init/template.sh | 2 +- .../knowstreaming-mysql/sql/ddl-ks-km.sql | 356 ------------------ .../knowstreaming-mysql/sql/ddl-logi-job.sql | 119 ------ .../sql/ddl-logi-security.sql | 205 ---------- .../knowstreaming-mysql/sql/dml-ks-km.sql | 7 - .../knowstreaming-mysql/sql/dml-logi.sql | 98 ----- 12 files changed, 1 insertion(+), 1356 deletions(-) delete mode 100644 km-dist/docker/knowstreaming-init/init/ks_kafka_broker_metric delete mode 100644 km-dist/docker/knowstreaming-init/init/ks_kafka_cluster_metric delete mode 100644 km-dist/docker/knowstreaming-init/init/ks_kafka_group_metric delete mode 100644 km-dist/docker/knowstreaming-init/init/ks_kafka_partition_metric delete mode 100644 km-dist/docker/knowstreaming-init/init/ks_kafka_replication_metric delete mode 100644 km-dist/docker/knowstreaming-init/init/ks_kafka_topic_metric delete mode 100644 km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql delete mode 100644 km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql delete mode 100644 km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql delete mode 100644 km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql delete mode 100644 km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql diff --git a/km-dist/docker/knowstreaming-init/init/ks_kafka_broker_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_broker_metric deleted file mode 100644 index 78933a4d..00000000 --- a/km-dist/docker/knowstreaming-init/init/ks_kafka_broker_metric +++ /dev/null @@ -1,102 +0,0 @@ -PUT _template/ks_kafka_broker_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_broker_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "NetworkProcessorAvgIdle" : { - "type" : "float" - }, - "UnderReplicatedPartitions" : { - "type" : "float" - }, - "BytesIn_min_15" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "RequestHandlerAvgIdle" : { - "type" : "float" - }, - "connectionsCount" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "TotalRequestQueueSize" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "TotalResponseQueueSize" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/ks_kafka_cluster_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_cluster_metric deleted file mode 100644 index 76d51cbd..00000000 --- a/km-dist/docker/knowstreaming-init/init/ks_kafka_cluster_metric +++ /dev/null @@ -1,187 +0,0 @@ -PUT _template/ks_kafka_cluster_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "Connections" : { - "type" : "double" - }, - "BytesIn_min_15" : { - "type" : "double" - }, - "PartitionURP" : { - "type" : "double" - }, - "HealthScore_Topics" : { - "type" : "double" - }, - "EventQueueSize" : { - "type" : "double" - }, - "ActiveControllerCount" : { - "type" : "double" - }, - "GroupDeads" : { - "type" : "double" - }, - "BytesIn_min_5" : { - "type" : "double" - }, - "HealthCheckTotal_Topics" : { - "type" : "double" - }, - "Partitions" : { - "type" : "double" - }, - "BytesOut" : { - "type" : "double" - }, - "Groups" : { - "type" : "double" - }, - "BytesOut_min_15" : { - "type" : "double" - }, - "TotalRequestQueueSize" : { - "type" : "double" - }, - "HealthCheckPassed_Groups" : { - "type" : "double" - }, - "TotalProduceRequests" : { - "type" : "double" - }, - "HealthCheckPassed" : { - "type" : "double" - }, - "TotalLogSize" : { - "type" : "double" - }, - "GroupEmptys" : { - "type" : "double" - }, - "PartitionNoLeader" : { - "type" : "double" - }, - "HealthScore_Brokers" : { - "type" : "double" - }, - "Messages" : { - "type" : "double" - }, - "Topics" : { - "type" : "double" - }, - "PartitionMinISR_E" : { - "type" : "double" - }, - "HealthCheckTotal" : { - "type" : "double" - }, - "Brokers" : { - "type" : "double" - }, - "Replicas" : { - "type" : "double" - }, - "HealthCheckTotal_Groups" : { - "type" : "double" - }, - "GroupRebalances" : { - "type" : "double" - }, - "MessageIn" : { - "type" : "double" - }, - "HealthScore" : { - "type" : "double" - }, - "HealthCheckPassed_Topics" : { - "type" : "double" - }, - "HealthCheckTotal_Brokers" : { - "type" : "double" - }, - "PartitionMinISR_S" : { - "type" : "double" - }, - "BytesIn" : { - "type" : "double" - }, - "BytesOut_min_5" : { - "type" : "double" - }, - "GroupActives" : { - "type" : "double" - }, - "MessagesIn" : { - "type" : "double" - }, - "GroupReBalances" : { - "type" : "double" - }, - "HealthCheckPassed_Brokers" : { - "type" : "double" - }, - "HealthScore_Groups" : { - "type" : "double" - }, - "TotalResponseQueueSize" : { - "type" : "double" - }, - "Zookeepers" : { - "type" : "double" - }, - "LeaderMessages" : { - "type" : "double" - }, - "HealthScore_Cluster" : { - "type" : "double" - }, - "HealthCheckPassed_Cluster" : { - "type" : "double" - }, - "HealthCheckTotal_Cluster" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/ks_kafka_group_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_group_metric deleted file mode 100644 index 9fe7008b..00000000 --- a/km-dist/docker/knowstreaming-init/init/ks_kafka_group_metric +++ /dev/null @@ -1,75 +0,0 @@ -PUT _template/ks_kafka_group_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_group_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "group" : { - "type" : "keyword" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "HealthScore" : { - "type" : "float" - }, - "Lag" : { - "type" : "float" - }, - "OffsetConsumed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "groupMetric" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/ks_kafka_partition_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_partition_metric deleted file mode 100644 index 2538eba3..00000000 --- a/km-dist/docker/knowstreaming-init/init/ks_kafka_partition_metric +++ /dev/null @@ -1,66 +0,0 @@ -PUT _template/ks_kafka_partition_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/ks_kafka_replication_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_replication_metric deleted file mode 100644 index b7beda54..00000000 --- a/km-dist/docker/knowstreaming-init/init/ks_kafka_replication_metric +++ /dev/null @@ -1,23 +0,0 @@ -PUT _template/ks_kafka_replication_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_replication_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/ks_kafka_topic_metric b/km-dist/docker/knowstreaming-init/init/ks_kafka_topic_metric deleted file mode 100644 index a2456dc6..00000000 --- a/km-dist/docker/knowstreaming-init/init/ks_kafka_topic_metric +++ /dev/null @@ -1,117 +0,0 @@ -PUT _template/ks_kafka_topic_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_topic_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "topic" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "BytesIn_min_15" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "BytesRejected" : { - "type" : "float" - }, - "PartitionURP" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ReplicationCount" : { - "type" : "float" - }, - "ReplicationBytesOut" : { - "type" : "float" - }, - "ReplicationBytesIn" : { - "type" : "float" - }, - "FailedFetchRequests" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "LogSize" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "FailedProduceRequests" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "brokerAgg" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/template.sh b/km-dist/docker/knowstreaming-init/init/template.sh index 43097279..2b091e10 100644 --- a/km-dist/docker/knowstreaming-init/init/template.sh +++ b/km-dist/docker/knowstreaming-init/init/template.sh @@ -1,7 +1,7 @@ echo "Wait ElasticSearch Start..." while true do - curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes >> /dev/null 2>&1 + curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 if [ "$?" != "0" ];then sleep 1s else diff --git a/km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql b/km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql deleted file mode 100644 index 50696917..00000000 --- a/km-dist/docker/knowstreaming-mysql/sql/ddl-ks-km.sql +++ /dev/null @@ -1,356 +0,0 @@ --- KS-KM自身的SQL，KS-KM依赖 Logi-Job 和 Logi-Security，因此另外两个ddl sql文件也需要执行 - -DROP TABLE IF EXISTS `ks_km_broker`; -CREATE TABLE `ks_km_broker` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '物理集群ID', - `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', - `host` varchar(128) NOT NULL DEFAULT '' COMMENT 'broker主机名', - `port` int(16) NOT NULL DEFAULT '-1' COMMENT 'broker端口', - `jmx_port` int(16) NOT NULL DEFAULT '-1' COMMENT 'Jmx端口', - `start_timestamp` bigint(20) NOT NULL DEFAULT '-1' COMMENT '启动时间', - `rack` varchar(128) NOT NULL DEFAULT '' COMMENT 'Rack信息', - `status` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1存活，0未存活', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `endpoint_map` varchar(1024) NOT NULL DEFAULT '' COMMENT '监听信息', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_phy_id_broker_id` (`cluster_phy_id`,`broker_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Broker信息表'; - - - -DROP TABLE IF EXISTS `ks_km_broker_config`; -CREATE TABLE `ks_km_broker_config` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', - `config_name` varchar(192) NOT NULL DEFAULT '' COMMENT '配置名称', - `config_value` text COMMENT '配置值', - `diff_type` int(16) NOT NULL DEFAULT '-1' COMMENT '差异类型', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_broker_name` (`cluster_phy_id`,`broker_id`,`config_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Broker配置信息表'; - - - -DROP TABLE IF EXISTS `ks_km_cluster_balance_job`; -CREATE TABLE `ks_km_cluster_balance_job` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', - `brokers` varchar(1024) NOT NULL DEFAULT '' COMMENT '均衡节点', - `topic_black_list` varchar(4096) NOT NULL DEFAULT '' COMMENT 'topic黑名单', - `type` int(16) NOT NULL DEFAULT '0' COMMENT '1:周期均衡，2：立即均衡', - `balance_interval_json` text COMMENT '均衡区间详情', - `metric_calculation_period` int(16) NOT NULL DEFAULT '0' COMMENT '指标计算周期，单位分钟', - `reassignment_json` text COMMENT '迁移脚本', - `parallel_num` int(16) NOT NULL DEFAULT '0' COMMENT '任务并行数', - `execution_strategy` int(16) NOT NULL DEFAULT '0' COMMENT '执行策略， 1：优先最大副本，2：优先最小副本', - `throttle_unit_b` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', - `total_reassign_size` double NOT NULL DEFAULT '0' COMMENT '总迁移大小', - `total_reassign_replica_num` int(16) NOT NULL DEFAULT '0' COMMENT '总迁移副本数', - `move_in_topic_list` varchar(4096) NOT NULL DEFAULT '' COMMENT '移入topic', - `broker_balance_detail` text COMMENT '节点均衡详情', - `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态 1：进行中，2：准备，3，成功，4：失败，5：取消', - `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', - `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务开始时间', - `finished_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务完成时间', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', - `description` text COMMENT '备注', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群均衡任务'; - - - -DROP TABLE IF EXISTS `ks_km_cluster_balance_job_config`; -CREATE TABLE `ks_km_cluster_balance_job_config` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', - `brokers` varchar(256) NOT NULL DEFAULT '' COMMENT '均衡节点', - `topic_black_list` varchar(4096) NOT NULL DEFAULT '' COMMENT 'topic黑名单', - `task_cron` varchar(64) NOT NULL DEFAULT '' COMMENT '任务周期', - `balance_interval_json` text COMMENT '均衡区间详情', - `metric_calculation_period` int(16) NOT NULL DEFAULT '0' COMMENT '指标计算周期，单位分钟', - `reassignment_json` text COMMENT '迁移脚本', - `parallel_num` int(16) NOT NULL DEFAULT '0' COMMENT '任务并行数', - `execution_strategy` int(16) NOT NULL DEFAULT '0' COMMENT '执行策略， 1：优先最大副本，2：优先最小副本', - `throttle_unit_b` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', - `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', - `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态 0：未开启，1：开启', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群均衡任务'; - - - -DROP TABLE IF EXISTS `ks_km_cluster_balance_reassign`; -CREATE TABLE `ks_km_cluster_balance_reassign` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `job_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '父任务ID', - `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', - `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', - `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', - `original_broker_ids` text COMMENT '源BrokerId列表', - `reassign_broker_ids` text COMMENT '目标BrokerId列表', - `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务开始时间', - `finished_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务完成时间', - `extend_data` text COMMENT '扩展数据', - `status` int(16) NOT NULL DEFAULT '2' COMMENT '任务状态', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群平衡迁移详情'; - - - -DROP TABLE IF EXISTS `ks_km_group_member`; -CREATE TABLE `ks_km_group_member` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `topic_name` varchar(192) NOT NULL DEFAULT '' COMMENT 'Topic名称', - `group_name` varchar(192) NOT NULL DEFAULT '' COMMENT 'Group名称', - `kafka_user` varchar(192) NOT NULL DEFAULT '' COMMENT 'Kafka用户', - `state` varchar(64) NOT NULL DEFAULT '' COMMENT '状态', - `member_count` int(11) NOT NULL DEFAULT '0' COMMENT '成员数', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_topic_group` (`cluster_phy_id`,`topic_name`,`group_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='GroupMember信息表'; - - - -DROP TABLE IF EXISTS `ks_km_health_check_result`; -CREATE TABLE `ks_km_health_check_result` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `dimension` int(11) NOT NULL DEFAULT '0' COMMENT '检查维度(0:未知，1:Cluster，2:Broker，3:Topic，4:Group)', - `config_name` varchar(192) NOT NULL DEFAULT '' COMMENT '配置名', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '物理集群ID', - `res_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', - `passed` int(11) NOT NULL DEFAULT '0' COMMENT '检查通过(0:未通过，1:通过)', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_dimension_config_cluster_res` (`dimension`,`config_name`,`cluster_phy_id`,`res_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='健康检查结果'; - - - -DROP TABLE IF EXISTS `ks_km_job`; -CREATE TABLE `ks_km_job` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', - `job_name` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务名称', - `job_type` tinyint(10) NOT NULL COMMENT '任务类型', - `job_status` tinyint(10) NOT NULL COMMENT '任务状态', - `job_data` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务的详细信息', - `job_desc` varchar(1024) NOT NULL DEFAULT '' COMMENT '任务描述', - `cluster_id` int(11) NOT NULL COMMENT 'kafka集群id', - `target` varchar(8192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务执行对象', - `running_status` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '任务运行详细状态(json), Success：7 Fail：1 Doing：2', - `creator` varchar(45) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '创建者', - `plan_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '计划执行时间', - `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '实际执行时间', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - KEY `index_cluster_id` (`cluster_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Job信息'; - - - -DROP TABLE IF EXISTS `ks_km_kafka_acl`; -CREATE TABLE `ks_km_kafka_acl` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', - `principal` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Kafka用户', - `operation` int(11) NOT NULL DEFAULT '0' COMMENT '操作', - `permission_type` int(11) NOT NULL DEFAULT '0' COMMENT '权限类型(0:未知，1:任意，2:拒绝，3:允许)', - `host` varchar(192) NOT NULL DEFAULT '127.0.0.1' COMMENT '机器', - `resource_type` int(11) NOT NULL DEFAULT '0' COMMENT '资源类型(0:未知，1:任意，2:Topic，3:Group，4:Cluster，5:事务，6:Token)', - `resource_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', - `pattern_type` int(11) NOT NULL COMMENT '匹配类型(0:未知，1:任意，2:Match，3:Literal，4:prefixed)', - `unique_field` varchar(1024) NOT NULL DEFAULT '' COMMENT '唯一字段，由cluster_phy_id等字段组成', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_uniq_field` (`unique_field`), - KEY `idx_cluster_phy_id_principal_res_name` (`cluster_phy_id`,`principal`,`resource_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='ACL信息'; - - - -DROP TABLE IF EXISTS `ks_km_kafka_change_record`; -CREATE TABLE `ks_km_kafka_change_record` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `res_type` int(11) NOT NULL DEFAULT '-1' COMMENT '资源类型', - `res_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', - `operate_type` int(11) NOT NULL DEFAULT '-1' COMMENT '操作类型', - `operate_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '操作时间', - `unique_field` varchar(1024) NOT NULL DEFAULT '' COMMENT '唯一键字段', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `unique_field` (`unique_field`), - KEY `idx_cluster_phy_id` (`cluster_phy_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Kafka变更记录表'; - - - -DROP TABLE IF EXISTS `ks_km_kafka_controller`; -CREATE TABLE `ks_km_kafka_controller` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群id', - `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', - `broker_host` varchar(256) NOT NULL DEFAULT '' COMMENT '主机名', - `broker_rack` varchar(256) NOT NULL DEFAULT '' COMMENT 'BrokerRack信息', - `timestamp` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'controller变更时间，-1表示未存活', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_broker_timestamp` (`cluster_phy_id`,`broker_id`,`timestamp`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='controller记录表'; - - - -DROP TABLE IF EXISTS `ks_km_kafka_user`; -CREATE TABLE `ks_km_kafka_user` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '名称', - `token` varchar(8192) NOT NULL DEFAULT '' COMMENT '密钥', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_phy_id_name` (`cluster_phy_id`,`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Kafka-User信息表'; - - - -DROP TABLE IF EXISTS `ks_km_partition`; -CREATE TABLE `ks_km_partition` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', - `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', - `leader_broker_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区的LeaderBroker，-1表示无Leader', - `in_sync_replicas` varchar(512) NOT NULL DEFAULT '-1' COMMENT 'ISR', - `assign_replicas` varchar(512) NOT NULL DEFAULT '-1' COMMENT 'AR', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_topic_partition` (`cluster_phy_id`,`topic_name`,`partition_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Partition信息表'; - - - -DROP TABLE IF EXISTS `ks_km_physical_cluster`; -CREATE TABLE `ks_km_physical_cluster` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '集群id', - `name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群名称', - `zookeeper` varchar(2048) NOT NULL DEFAULT '' COMMENT 'zk地址', - `bootstrap_servers` varchar(2048) NOT NULL DEFAULT '' COMMENT 'server地址', - `kafka_version` varchar(32) NOT NULL DEFAULT '' COMMENT 'kafka版本', - `client_properties` text COMMENT 'Kafka客户端配置', - `jmx_properties` text COMMENT 'JMX配置', - `description` text COMMENT '备注', - `auth_type` int(11) NOT NULL DEFAULT '0' COMMENT '认证类型，-1未知，0:无认证，', - `run_state` tinyint(4) NOT NULL DEFAULT '1' COMMENT '运行状态, 0表示未监控, 1监控中，有ZK，2:监控中，无ZK', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '接入时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_name` (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='物理集群信息表'; - - - -DROP TABLE IF EXISTS `ks_km_platform_cluster_config`; -CREATE TABLE `ks_km_platform_cluster_config` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', - `cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `value_group` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项组', - `value_name` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项名字', - `value` text COMMENT '配置项的值', - `description` text COMMENT '备注', - `operator` varchar(16) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '操作者', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_id_group_name` (`cluster_id`,`value_group`,`value_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='KS平台集群配置'; - - - -DROP TABLE IF EXISTS `ks_km_reassign_job`; -CREATE TABLE `ks_km_reassign_job` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', - `reassignment_json` text COMMENT '迁移计划', - `throttle_unit_byte` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', - `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务开始时间', - `finished_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务完成时间', - `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', - `description` text COMMENT '备注', - `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='迁移Job信息'; - - - -DROP TABLE IF EXISTS `ks_km_reassign_sub_job`; -CREATE TABLE `ks_km_reassign_sub_job` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', - `job_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '父任务ID', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', - `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', - `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', - `original_broker_ids` text COMMENT '源BrokerId列表', - `reassign_broker_ids` text COMMENT '目标BrokerId列表', - `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务开始时间', - `finished_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务完成时间', - `extend_data` text COMMENT '扩展数据', - `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='迁移SubJob信息'; - - - -DROP TABLE IF EXISTS `ks_km_topic`; -CREATE TABLE `ks_km_topic` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', - `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', - `replica_num` int(11) NOT NULL DEFAULT '-1' COMMENT '副本数', - `partition_num` int(11) NOT NULL DEFAULT '-1' COMMENT '分区数', - `broker_ids` varchar(2048) NOT NULL DEFAULT '' COMMENT 'BrokerId列表', - `partition_map` text COMMENT '分区分布信息', - `retention_ms` bigint(20) NOT NULL DEFAULT '-2' COMMENT '保存时间，-2：未知，-1：无限制，>=0对应时间，单位ms', - `type` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'Topic类型，默认0，0:普通，1:Kafka内部', - `description` text COMMENT '备注信息', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间(尽量与Topic实际创建时间一致)', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间(尽量与Topic实际创建时间一致)', - PRIMARY KEY (`id`), - UNIQUE KEY `uniq_cluster_phy_id_topic_name` (`cluster_phy_id`,`topic_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Topic信息表'; - - -DROP TABLE IF EXISTS `ks_km_app_node`; -CREATE TABLE `ks_km_app_node` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', - `host_name` varchar(256) NOT NULL DEFAULT '' COMMENT 'host', - `ip` varchar(256) NOT NULL DEFAULT '' COMMENT 'ip', - `beat_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'node 的心跳时间', - `app_name` varchar(128) NOT NULL DEFAULT '' COMMENT 'km 集群的应用名', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - KEY `idx_app_host` (`app_name`,`host_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='km集群部署的node信息'; diff --git a/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql b/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql deleted file mode 100644 index 3ba22983..00000000 --- a/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-job.sql +++ /dev/null @@ -1,119 +0,0 @@ --- Logi-Job模块的sql，安装KS-KM需要执行该sql - - -DROP TABLE IF EXISTS `logi_job`; -CREATE TABLE `logi_job` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `job_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', - `task_code` varchar(255) NOT NULL DEFAULT '' COMMENT '任务code', - `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', - `try_times` int(10) NOT NULL DEFAULT '0' COMMENT '第几次重试', - `worker_code` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器', - `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', - `start_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '开始时间', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `job_code` (`job_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='正在执行的job信息'; - - - -DROP TABLE IF EXISTS `logi_job_log`; -CREATE TABLE `logi_job_log` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `job_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'job taskCode', - `task_code` varchar(255) NOT NULL DEFAULT '' COMMENT '任务code', - `task_name` varchar(255) NOT NULL DEFAULT '' COMMENT '任务名称', - `task_desc` varchar(255) NOT NULL DEFAULT '' COMMENT '任务描述', - `task_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '任务id', - `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', - `try_times` int(10) NOT NULL DEFAULT '0' COMMENT '第几次重试', - `worker_code` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器', - `worker_ip` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器ip', - `start_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '开始时间', - `end_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '结束时间', - `status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '执行结果 1成功 2失败 3取消', - `error` text NOT NULL COMMENT '错误信息', - `result` text NOT NULL COMMENT '执行结果', - `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - KEY `index_job_code` (`job_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='job执行历史日志'; - - - -DROP TABLE IF EXISTS `logi_task`; -CREATE TABLE `logi_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `task_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', - `task_name` varchar(255) NOT NULL DEFAULT '' COMMENT '名称', - `task_desc` varchar(1000) NOT NULL DEFAULT '' COMMENT '任务描述', - `cron` varchar(100) NOT NULL DEFAULT '' COMMENT 'cron 表达式', - `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', - `params` varchar(1000) NOT NULL DEFAULT '' COMMENT '执行参数 map 形式{key1:value1,key2:value2}', - `retry_times` int(10) NOT NULL DEFAULT '0' COMMENT '允许重试次数', - `last_fire_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '上次执行时间', - `timeout` bigint(20) NOT NULL DEFAULT '0' COMMENT '超时 毫秒', - `status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '1等待 2运行中 3暂停', - `sub_task_codes` varchar(1000) NOT NULL DEFAULT '' COMMENT '子任务code列表,逗号分隔', - `consensual` varchar(200) NOT NULL DEFAULT '' COMMENT '执行策略', - `owner` varchar(200) NOT NULL DEFAULT '' COMMENT '责任人', - `task_worker_str` varchar(3000) NOT NULL DEFAULT '' COMMENT '机器执行信息', - `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `task_code` (`task_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='任务信息'; - - - -DROP TABLE IF EXISTS `logi_task_lock`; -CREATE TABLE `logi_task_lock` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `task_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', - `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', - `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', - `expire_time` bigint(20) NOT NULL DEFAULT '0' COMMENT '过期时间', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='任务锁'; - - - -DROP TABLE IF EXISTS `logi_worker`; -CREATE TABLE `logi_worker` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', - `worker_name` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker名', - `ip` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker的ip', - `cpu` int(11) NOT NULL DEFAULT '0' COMMENT 'cpu数量', - `cpu_used` double NOT NULL DEFAULT '0' COMMENT 'cpu使用率', - `memory` double NOT NULL DEFAULT '0' COMMENT '内存,以M为单位', - `memory_used` double NOT NULL DEFAULT '0' COMMENT '内存使用率', - `jvm_memory` double NOT NULL DEFAULT '0' COMMENT 'jvm堆大小，以M为单位', - `jvm_memory_used` double NOT NULL DEFAULT '0' COMMENT 'jvm堆使用率', - `job_num` int(10) NOT NULL DEFAULT '0' COMMENT '正在执行job数', - `heartbeat` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '心跳时间', - `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `worker_code` (`worker_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker信息'; - - - -DROP TABLE IF EXISTS `logi_worker_blacklist`; -CREATE TABLE `logi_worker_blacklist` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `worker_code` (`worker_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker黑名单列表'; diff --git a/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql b/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql deleted file mode 100644 index 7efa1f3a..00000000 --- a/km-dist/docker/knowstreaming-mysql/sql/ddl-logi-security.sql +++ /dev/null @@ -1,205 +0,0 @@ -#-----------------------创建表----------------------- -DROP TABLE IF EXISTS `logi_security_dept`; -CREATE TABLE `logi_security_dept` -( - id int auto_increment primary key, - dept_name varchar(10) not null comment '部门名', - parent_id int not null comment '父部门id', - leaf tinyint(1) not null comment '是否叶子部门', - level tinyint not null comment 'parentId为0的层级为1', - description varchar(20) null comment '描述', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '部门信息表'; - -DROP TABLE IF EXISTS `logi_security_message`; -CREATE TABLE `logi_security_message` -( - id int auto_increment primary key, - title varchar(60) not null comment '标题', - content varchar(256) null comment '内容', - read_tag tinyint(1) default 0 null comment '是否已读', - oplog_id int null comment '操作日志id', - user_id int null comment '这条消息属于哪个用户的，用户id', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '消息中心'; - -DROP TABLE IF EXISTS `logi_security_oplog`; -CREATE TABLE `logi_security_oplog` -( - id int auto_increment primary key, - operator_ip varchar(20) not null comment '操作者ip', - operator varchar(20) null comment '操作者账号', - operate_page varchar(16) not null default '' comment '操作页面', - operate_type varchar(16) not null comment '操作类型', - target_type varchar(16) not null comment '对象分类', - target varchar(128) not null comment '操作对象', - operation_methods varchar(20) not null default '' comment '操作方式', - detail text null comment '日志详情', - create_time timestamp default CURRENT_TIMESTAMP null, - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '操作日志'; - -DROP TABLE IF EXISTS `logi_security_oplog_extra`; -CREATE TABLE `logi_security_oplog_extra` -( - id int auto_increment primary key, - info varchar(16) null comment '信息', - type tinyint not null comment '哪种信息：1：操作页面;2：操作类型;3：对象分类', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '操作日志信息（操作页面、操作类型、对象分类）'; - -DROP TABLE IF EXISTS `logi_security_permission`; -CREATE TABLE `logi_security_permission` -( - id int auto_increment primary key, - permission_name varchar(40) not null comment '权限名字', - parent_id int not null comment '父权限id', - leaf tinyint(1) not null comment '是否叶子权限点（具体的操作）', - level tinyint not null comment '权限点的层级（parentId为0的层级为1）', - description varchar(64) null comment '权限点描述', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '权限表'; - -DROP TABLE IF EXISTS `logi_security_project`; -CREATE TABLE `logi_security_project` -( - id int auto_increment comment '项目id' primary key, - project_code varchar(128) not null comment '项目编号', - project_name varchar(128) not null comment '项目名', - description varchar(512) default '' not null comment '项目描述', - dept_id int not null comment '部门id', - running tinyint(1) default 1 not null comment '启用 or 停用', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '项目表'; - -DROP TABLE IF EXISTS `logi_security_resource_type`; -CREATE TABLE `logi_security_resource_type` -( - id int auto_increment primary key, - type_name varchar(16) null comment '资源类型名', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '资源类型表'; - -DROP TABLE IF EXISTS `logi_security_role`; -CREATE TABLE `logi_security_role` -( - id int auto_increment primary key, - role_code varchar(128) not null comment '角色编号', - role_name varchar(128) not null comment '名称', - description varchar(128) null comment '角色描述', - last_reviser varchar(30) null comment '最后修改人', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '角色信息'; - -DROP TABLE IF EXISTS `logi_security_role_permission`; -CREATE TABLE `logi_security_role_permission` -( - id int auto_increment primary key, - role_id int not null comment '角色id', - permission_id int not null comment '权限id', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '角色权限表（只保留叶子权限与角色关系）'; - -DROP TABLE IF EXISTS `logi_security_user`; -CREATE TABLE `logi_security_user` -( - id int auto_increment primary key, - user_name varchar(64) not null comment '用户账号', - pw varchar(2048) not null comment '用户密码', - salt char(5) default '' not null comment '密码盐', - real_name varchar(128) default '' not null comment '真实姓名', - phone char(11) default '' not null comment 'mobile', - email varchar(30) default '' not null comment 'email', - dept_id int null comment '所属部门id', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - create_time timestamp default CURRENT_TIMESTAMP null comment '注册时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户信息'; - -DROP TABLE IF EXISTS `logi_security_user_project`; -CREATE TABLE `logi_security_user_project` -( - id int auto_increment primary key, - user_id int not null comment '用户id', - project_id int not null comment '项目id', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户项目关系表（项目负责人）'; - -DROP TABLE IF EXISTS `logi_security_user_resource`; -CREATE TABLE `logi_security_user_resource` -( - id int auto_increment primary key, - user_id int not null comment '用户id', - project_id int not null comment '资源所属项目id', - resource_type_id int not null comment '资源类别id', - resource_id int not null comment '资源id', - control_level tinyint not null comment '管理级别：1（查看权限）2（管理权限）', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户和资源关系表'; - -DROP TABLE IF EXISTS `logi_security_user_role`; -CREATE TABLE `logi_security_user_role` -( - id int auto_increment primary key, - user_id int not null comment '用户id', - role_id int not null comment '角色id', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', - is_delete tinyint(1) default 0 not null comment '逻辑删除', - app_name varchar(16) null comment '应用名称' -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户角色表'; - --- ---------------------------- --- Table structure for logi_config --- ---------------------------- -DROP TABLE IF EXISTS `logi_security_config`; -CREATE TABLE `logi_security_config` -( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', - `value_group` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项组', - `value_name` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项名字', - `value` text COMMENT '配置项的值', - `edit` int(4) NOT NULL DEFAULT '1' COMMENT '是否可以编辑 1 不可编辑（程序获取） 2 可编辑', - `status` int(4) NOT NULL DEFAULT '1' COMMENT '1 正常 2 禁用', - `memo` varchar(1000) NOT NULL DEFAULT '' COMMENT '备注', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `is_delete` tinyint(1) NOT NULL DEFAULT '0' COMMENT '逻辑删除', - `app_name` varchar(16) COLLATE utf8_bin DEFAULT NULL COMMENT '应用名称', - `operator` varchar(16) COLLATE utf8_bin DEFAULT NULL COMMENT '操作者', - PRIMARY KEY (`id`), - KEY `idx_group_name` (`value_group`,`value_name`) -) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 COMMENT='logi配置项'; \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql b/km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql deleted file mode 100644 index 2d354a87..00000000 --- a/km-dist/docker/knowstreaming-mysql/sql/dml-ks-km.sql +++ /dev/null @@ -1,7 +0,0 @@ --- 检查检查配置 -INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_CLUSTER_NO_CONTROLLER','{ \"value\": 1, \"weight\": 30 } ','集群Controller数正常','know-streaming'); -INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_BROKER_REQUEST_QUEUE_FULL','{ \"value\": 10, \"weight\": 20 } ','Broker-RequestQueueSize指标','know-streaming'); -INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_BROKER_NETWORK_PROCESSOR_AVG_IDLE_TOO_LOW','{ \"value\": 0.8, \"weight\": 20 } ','Broker-NetworkProcessorAvgIdlePercent指标','know-streaming'); -INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_GROUP_RE_BALANCE_TOO_FREQUENTLY','{\n \"latestMinutes\": 10,\n \"detectedTimes\": 8,\n \"weight\": 10\n}\n','Group的re-balance频率','know-streaming'); -INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_TOPIC_NO_LEADER','{ \"value\": 1, \"weight\": 10 } ','Topic 无Leader数','know-stream'); -INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_TOPIC_UNDER_REPLICA_TOO_LONG','{ \"latestMinutes\": 10, \"detectedTimes\": 8, \"weight\": 10 } ','Topic 未同步持续时间','know-streaming'); diff --git a/km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql b/km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql deleted file mode 100644 index 927a0bcc..00000000 --- a/km-dist/docker/knowstreaming-mysql/sql/dml-logi.sql +++ /dev/null @@ -1,98 +0,0 @@ --- 初始化权限 -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1593', '多集群管理', '0', '0', '1', '多集群管理', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1595', '系统管理', '0', '0', '1', '系统管理', '0', 'know-streaming'); - --- 多集群管理权限 -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1597', '接入集群', '1593', '1', '2', '接入集群', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1599', '删除集群', '1593', '1', '2', '删除集群', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1601', 'Cluster-修改集群信息', '1593', '1', '2', 'Cluster-修改集群信息', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1603', 'Cluster-修改健康规则', '1593', '1', '2', 'Cluster-修改健康规则', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1605', 'Broker-修改Broker配置', '1593', '1', '2', 'Broker-修改Broker配置', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1607', 'Topic-新增Topic', '1593', '1', '2', 'Topic-新增Topic', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1609', 'Topic-扩分区', '1593', '1', '2', 'Topic-扩分区', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1611', 'Topic-删除Topic', '1593', '1', '2', 'Topic-扩分区', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1613', 'Topic-重置Offset', '1593', '1', '2', 'Topic-重置Offset', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1615', 'Topic-修改Topic配置', '1593', '1', '2', 'Topic-修改Topic配置', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1617', 'Consumers-重置Offset', '1593', '1', '2', 'Consumers-重置Offset', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1619', 'Test-Producer', '1593', '1', '2', 'Test-Producer', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1621', 'Test-Consumer', '1593', '1', '2', 'Test-Consumer', '0', 'know-streaming'); - --- 系统管理权限 -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1623', '配置管理-新增配置', '1595', '1', '2', '配置管理-新增配置', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1625', '配置管理-编辑配置', '1595', '1', '2', '配置管理-编辑配置', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1627', '配置管理-删除配置', '1595', '1', '2', '配置管理-删除配置', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1629', '用户管理-新增人员', '1595', '1', '2', '用户管理-新增人员', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1631', '用户管理-编辑人员', '1595', '1', '2', '用户管理-编辑人员', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1633', '用户管理-修改人员密码', '1595', '1', '2', '用户管理-修改人员密码', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1635', '用户管理-删除人员', '1595', '1', '2', '用户管理-删除人员', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1637', '用户管理-新增角色', '1595', '1', '2', '用户管理-新增角色', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1639', '用户管理-编辑角色', '1595', '1', '2', '用户管理-编辑角色', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1641', '用户管理-分配用户角色', '1595', '1', '2', '用户管理-分配用户角色', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1643', '用户管理-删除角色', '1595', '1', '2', '用户管理-删除角色', '0', 'know-streaming'); - --- 多集群管理权限2022-09-06新增 -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2000', '多集群管理查看', '1593', '1', '2', '多集群管理查看', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2002', 'Topic-迁移副本', '1593', '1', '2', 'Topic-迁移副本', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2004', 'Topic-扩缩副本', '1593', '1', '2', 'Topic-扩缩副本', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2006', 'Cluster-LoadReBalance-周期均衡', '1593', '1', '2', 'Cluster-LoadReBalance-周期均衡', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2008', 'Cluster-LoadReBalance-立即均衡', '1593', '1', '2', 'Cluster-LoadReBalance-立即均衡', '0', 'know-streaming'); -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2010', 'Cluster-LoadReBalance-设置集群规格', '1593', '1', '2', 'Cluster-LoadReBalance-设置集群规格', '0', 'know-streaming'); - - --- 系统管理权限2022-09-06新增 -INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('3000', '系统管理查看', '1595', '1', '2', '系统管理查看', '0', 'know-streaming'); - - - - - - --- 初始化用户 -INSERT INTO `logi_security_user` (`id`, `user_name`, `pw`, `real_name`, `is_delete`, `app_name`) VALUES ('1', 'admin', 'V1ZkU2RHRlhOSGxOUkVsNVdETjBRVlp0Y0V0T1IwWnlaVEZ6YWxGRVJrRkpNVEU1VTJwYVUySkhlRzlSU0RBOWUwQldha28wWVd0N1d5TkFNa0FqWFgxS05sSnNiR2hBZlE9PXtAVmpKNGFre1sjQDNAI119SjZSbGxoQH0=Mv{#cdRgJ45Lqx}3IubEW87!==', '系统管理员', '0', 'know-streaming'); - --- 初始化角色 -INSERT INTO `logi_security_role` (`id`, `role_code`, `role_name`, `description`, `last_reviser`, `is_delete`, `app_name`) VALUES ('1677', 'r15477137', '管理员角色', '包含系统所有权限', 'admin', '0', 'know-streaming'); - --- 初始化角色权限关系 -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1597', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1599', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1601', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1603', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1605', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1607', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1609', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1611', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1613', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1615', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1617', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1619', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1621', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1593', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1623', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1625', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1627', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1629', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1631', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1633', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1635', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1637', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1639', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1641', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1643', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1595', '0', 'know-streaming'); - -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2000', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2002', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2004', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2006', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2008', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2010', '0', 'know-streaming'); -INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '3000', '0', 'know-streaming'); - --- 初始化 用户角色关系 -INSERT INTO `logi_security_user_role` (`id`, `user_id`, `role_id`, `is_delete`, `app_name`) VALUES ('1', '1', '1677', '0', 'know-streaming'); - -INSERT INTO `logi_security_config` -(`value_group`,`value_name`,`value`,`edit`,`status`,`memo`,`is_delete`,`app_name`,`operator`) -VALUES -('SECURITY.LOGIN','SECURITY.TRICK_USERS','[\n \"admin\"\n]',1,1,'允许跳过登录的用户',0,'know-streaming','admin'); \ No newline at end of file From 227479f695944612c05ba15ddbb4c0bc74fdae86 Mon Sep 17 00:00:00 2001 From: _haoqi <1148648445@qq.com> Date: Fri, 16 Sep 2022 15:13:18 +0800 Subject: [PATCH 06/18] =?UTF-8?q?1.=20=E4=BF=AE=E6=94=B9dockerfile=202.=20?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 4 +- km-dist/docker/knowstreaming-init/Dockerfile | 10 +-- .../docker/knowstreaming-manage/Dockerfile | 38 ++++++-- km-dist/docker/knowstreaming-mysql/Dockerfile | 4 +- km-dist/docker/knowstreaming-web/Dockerfile | 90 +++++++++++++++++-- .../knowstreaming-web/conf/knowStreaming.conf | 46 ---------- 6 files changed, 119 insertions(+), 73 deletions(-) delete mode 100644 km-dist/docker/knowstreaming-web/conf/knowStreaming.conf diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index 446129ec..62d15324 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: knowstreaming-manager: - image: ks_manage:1.0 + image: km_manage:1.1 container_name: knowstreaming-manager privileged: true restart: always @@ -32,7 +32,7 @@ services: # volumes: # - /Users/haoqi/Downloads/ks/manage/log:/logs knowstreaming-ui: - image: ks_ui:1.0 + image: km:1.0 container_name: knowstreaming-ui restart: always ports: diff --git a/km-dist/docker/knowstreaming-init/Dockerfile b/km-dist/docker/knowstreaming-init/Dockerfile index 8f47558c..8dfec0a5 100644 --- a/km-dist/docker/knowstreaming-init/Dockerfile +++ b/km-dist/docker/knowstreaming-init/Dockerfile @@ -1,9 +1,7 @@ -FROM debian:stable-20220912-slim +FROM centos:7.9.2009 LABEL org.opencontainers.image.authors="haoqi123" -ENV INIT_ES_FILE=./init/* +ARG INIT_ES_FILE=./init/* COPY ${INIT_ES_FILE} / +ENV TZ=Asia/Shanghai -RUN apt-get update \ - && apt-get install -y curl - -ENTRYPOINT ["/bin/bash","/template.sh","start"] \ No newline at end of file +ENTRYPOINT ["/bin/sh","/template.sh","start"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index c73fd07d..fbfc2edf 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -1,14 +1,36 @@ -FROM amd64/openjdk:17.0.2 +FROM docker.io/amd64/eclipse-temurin:17.0.4.1_1-jdk-centos7 -LABEL org.opencontainers.image.authors="haoqi123" +LABEL maintainer="haoqi123 <1148648445@qq.com>" -EXPOSE 80 +WORKDIR /knowstreaming + +ARG VERSION_MAVEN=3.6.3 +ARG VERSION_KM=v3.0.0-beta.2 +ARG INIT_FILE=./init/* +ARG CONFIG_FILE=./conf/* -ENV JAR_FILE=./jar/*.jar -ENV INIT_FILE=./init/* -ENV CONFIG_FILE=./conf/* -COPY ${JAR_FILE} /app/app.jar COPY ${INIT_FILE} / COPY ${CONFIG_FILE} /conf/ -ENTRYPOINT ["/bin/bash","/start.sh"] \ No newline at end of file +# SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN yum install -y git + +RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_MAVEN}/binaries/apache-maven-${VERSION_MAVEN}-bin.tar.gz" \ + && mkdir maven/ \ + && tar -zxf "apache-maven-${VERSION_MAVEN}-bin.tar.gz" -C maven/ --strip-components=1 --no-same-owner \ + && rm -rf /knowstreaming/*.tar.gz \ + && ln -s /knowstreaming/maven/bin/mvn /usr/local/bin/mvn \ +# && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \ + && git clone https://github.com/didi/KnowStreaming.git \ +# && unzip -o -q master.zip \ + && cd KnowStreaming && git checkout ${VERSION_KM} \ + && sed -i "/km-console/d" ./pom.xml \ + && mvn -U clean package -Dmaven.test.skip=true \ + && cd / \ + && mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \ + && rm -rf /knowstreaming/* /usr/local/bin/mvn \ + && yum erase -y git && yum clean all \ + && rm -rf `pwd`/.m2 /tmp/* + +ENTRYPOINT ["/bin/bash","/start.sh"] diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile index 4e6917c9..95720f05 100644 --- a/km-dist/docker/knowstreaming-mysql/Dockerfile +++ b/km-dist/docker/knowstreaming-mysql/Dockerfile @@ -4,7 +4,7 @@ LABEL org.opencontainers.image.authors="haoqi123" EXPOSE 3306 -ENV INIT_SOURCE_FILE=./sql/*.sql -ENV MYSQL_CNF_FILE=./conf/*.cnf +ARG INIT_SOURCE_FILE=./sql/*.sql +ARG MYSQL_CNF_FILE=./conf/*.cnf COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/ COPY ${MYSQL_CNF_FILE} /etc/mysql/conf.d/ \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-web/Dockerfile b/km-dist/docker/knowstreaming-web/Dockerfile index 4ec52a71..ef7e6b78 100644 --- a/km-dist/docker/knowstreaming-web/Dockerfile +++ b/km-dist/docker/knowstreaming-web/Dockerfile @@ -1,14 +1,86 @@ -FROM docker.io/library/nginx:1.23.1 +FROM docker.io/amd64/nginx:1.22 -LABEL org.opencontainers.image.authors="haoqi123" +LABEL maintainer="haoqi123 <1148648445@qq.com>" -WORKDIR /pub -EXPOSE 80 +WORKDIR /knowstreaming -ENV HTML_FILE=./pub -ENV NGINX_CONF=./conf/*.conf +ARG VERSION_NODE=v14.20.0 +ARG VERSION_KM=v3.0.0-beta.2 -COPY ${NGINX_CONF} /etc/nginx/conf.d/default.conf -COPY ${HTML_FILE} /pub +# SHELL ["/bin/bash", "-o", "pipefail", "-c"] -HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 \ No newline at end of file +RUN apt update \ + && apt install -y git unzip + +RUN echo 'server { \ + listen 80; \ + server_name localhost; \ + gzip on; \ + gzip_buffers 16 8k; \ + gzip_comp_level 4; \ + gzip_http_version 1.0; \ + gzip_min_length 1280; \ + gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; \ + gzip_vary on; \ + root /pub; \ + location / { \ + root /pub; \ + if ($request_filename ~* .*\.(?:htm|html|json)$) { \ + add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; \ + } \ + try_files $uri /layout/index.html; \ + } \ + location ~* \.(json)$ { \ + add_header Cache-Control no-cache; \ + } \ + location @kmfallback { \ + } \ + #location ~ ^/(clusters|config|cluster|login) { \ + # rewrite ^.*$ /; \ + #} \ + location ~ ^/ks-km/api/v3 { \ + #rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; \ + proxy_pass http://knowstreaming-manager; \ + #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \ + #proxy_cookie_path /ks-km/ /; \ + #proxy_set_header Host $host; \ + #proxy_set_header Referer $http_referer; \ + #proxy_set_header Cookie $http_cookie; \ + #proxy_set_header X-Real-Ip $remote_addr; \ + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; \ + } \ + location ~ ^/logi-security/api/v1 { \ + #rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; \ + proxy_pass http://knowstreaming-manager; \ + #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \ + } \ + location ~ ^/(401|403|404|500){ \ + rewrite ^.*$ /; \ + } \ + }' > /etc/nginx/conf.d/default.conf + +RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/${VERSION_NODE}/node-${VERSION_NODE}-linux-x64.tar.gz" \ + && mkdir node/ \ + && tar -zxf "node-${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \ + && rm -rf /knowstreaming/*.tar.gz \ + && ln -s /knowstreaming/node/bin/node /usr/local/bin/node \ + && ln -s /knowstreaming/node/bin/npm /usr/local/bin/npm \ + && npm config set strict-ssl false \ + && npm config set registry https://registry.npm.taobao.org \ + && npm info underscore \ + && node --version \ + && npm --version \ + && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \ + && unzip -o -q master.zip \ + && cd KnowStreaming-master/km-console \ + && npm install -g lerna && npm run i && npm run build \ + && mv /knowstreaming/KnowStreaming-master/km-rest/src/main/resources/templates /pub \ + && npm uninstall -g lerna \ + && cd / \ + && rm -rf /knowstreaming/* /usr/local/bin/node /usr/local/bin/npm \ + && apt remove -y git unzip && apt autoremove -y && apt remove --purge --auto-remove -y \ + && rm -rf /var/lib/apt/lists/* /tmp /root/.npm + +#ENTRYPOINT ["ls", "/ks_ui/km-console"] + +#CMD ["nginx", "-g", "daemon off;"] diff --git a/km-dist/docker/knowstreaming-web/conf/knowStreaming.conf b/km-dist/docker/knowstreaming-web/conf/knowStreaming.conf deleted file mode 100644 index 9f5007ff..00000000 --- a/km-dist/docker/knowstreaming-web/conf/knowStreaming.conf +++ /dev/null @@ -1,46 +0,0 @@ -server { - listen 80; - server_name localhost; - gzip on; - gzip_buffers 16 8k; - gzip_comp_level 4; - gzip_http_version 1.0; - gzip_min_length 1280; - gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; - gzip_vary on; - root /pub; - location / { - root /pub; - if ($request_filename ~* .*\.(?:htm|html|json)$) { - add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; - } - try_files $uri /layout/index.html; - } - location ~* \.(json)$ { - add_header Cache-Control no-cache; - } - location @kmfallback { - } - #location ~ ^/(clusters|config|cluster|login) { - # rewrite ^.*$ /; - #} - location ~ ^/ks-km/api/v3 { - #rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; - proxy_pass http://knowstreaming-manager; - #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; - #proxy_cookie_path /ks-km/ /; - #proxy_set_header Host $host; - #proxy_set_header Referer $http_referer; - #proxy_set_header Cookie $http_cookie; - #proxy_set_header X-Real-Ip $remote_addr; - #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - location ~ ^/logi-security/api/v1 { - #rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; - proxy_pass http://knowstreaming-manager; - #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; - } - location ~ ^/(401|403|404|500){ - rewrite ^.*$ /; - } - } \ No newline at end of file From 7b0b6936e0dc16ff77569675a15aba26c784febc Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Fri, 16 Sep 2022 15:54:34 +0800 Subject: [PATCH 07/18] =?UTF-8?q?1.=20=E8=B0=83=E6=95=B4docker-compose.yml?= =?UTF-8?q?=E4=B8=AD=E5=AE=B9=E5=99=A8=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 34 ++++++++----------- km-dist/docker/knowstreaming-init/Dockerfile | 7 ++-- .../knowstreaming-init/init/template.sh | 2 +- .../docker/knowstreaming-manage/Dockerfile | 2 +- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index 62d15324..1e736d31 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -3,22 +3,22 @@ version: "3" services: knowstreaming-manager: - image: km_manage:1.1 + image: knowstreaming-manager:0.1.0 container_name: knowstreaming-manager privileged: true restart: always depends_on: - elasticsearch_single - - kmysql + - knowstreaming_mysql links: - elasticsearch_single - - kmysql + - knowstreaming_mysql expose: - 80 environment: TZ: Asia/Shanghai - SERVER_MYSQL_ADDRESS: kmysql:3306 + SERVER_MYSQL_ADDRESS: knowstreaming_mysql:3306 SERVER_MYSQL_DB: know_streaming SERVER_MYSQL_USER: root SERVER_MYSQL_PASSWORD: admin2022_ @@ -28,11 +28,10 @@ services: JAVA_OPTS: -Xmx1g -Xms1g networks: - ks -# 日志文件持久化路径 # volumes: # - /Users/haoqi/Downloads/ks/manage/log:/logs knowstreaming-ui: - image: km:1.0 + image: knowstreaming-ui:0.1.0 container_name: knowstreaming-ui restart: always ports: @@ -54,23 +53,20 @@ services: expose: - 9200 - 9300 -# 对外访问 -# ports: -# - '9200:9200' -# - '9300:9300' + ports: + - '9200:9200' + - '9300:9300' environment: TZ: Asia/Shanghai ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node -# 数据文件持久化路径 -# 记录实时指标信息 # volumes: # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data networks: - ks - esinit: - image: esinit:1.0 - container_name: esinit + knowstreaming_init: + image: knowstreaming-init:0.1.0 + container_name: knowstreaming_init depends_on: - elasticsearch_single links: @@ -81,9 +77,9 @@ services: networks: - ks - kmysql: - image: ks_mysql:1.0 - container_name: kmysql + knowstreaming_mysql: + image: knowstreaming-mysql:0.1.0 + container_name: knowstreaming_mysql restart: always environment: TZ: Asia/Shanghai @@ -93,8 +89,6 @@ services: - ks expose: - 3306 -# 数据文件持久化路径 -# 记录添加的集群基本信息 # volumes: # - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql diff --git a/km-dist/docker/knowstreaming-init/Dockerfile b/km-dist/docker/knowstreaming-init/Dockerfile index 8dfec0a5..ddb2120c 100644 --- a/km-dist/docker/knowstreaming-init/Dockerfile +++ b/km-dist/docker/knowstreaming-init/Dockerfile @@ -1,7 +1,10 @@ -FROM centos:7.9.2009 +FROM debian:stable-20220912-slim LABEL org.opencontainers.image.authors="haoqi123" ARG INIT_ES_FILE=./init/* COPY ${INIT_ES_FILE} / ENV TZ=Asia/Shanghai -ENTRYPOINT ["/bin/sh","/template.sh","start"] \ No newline at end of file +RUN apt-get update \ + && apt-get install -y curl + +ENTRYPOINT ["/bin/bash","/template.sh","start"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/template.sh b/km-dist/docker/knowstreaming-init/init/template.sh index 2b091e10..ce41db73 100644 --- a/km-dist/docker/knowstreaming-init/init/template.sh +++ b/km-dist/docker/knowstreaming-init/init/template.sh @@ -1,4 +1,4 @@ -echo "Wait ElasticSearch Start..." +echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}" while true do curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index fbfc2edf..95c20f7a 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -31,6 +31,6 @@ RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_M && mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \ && rm -rf /knowstreaming/* /usr/local/bin/mvn \ && yum erase -y git && yum clean all \ - && rm -rf `pwd`/.m2 /tmp/* + && rm -rf /root/.m2 /tmp/* ENTRYPOINT ["/bin/bash","/start.sh"] From 121f8468d5282e6cd4aa03677d85e38ebef5b81e Mon Sep 17 00:00:00 2001 From: _haoqi <1148648445@qq.com> Date: Fri, 16 Sep 2022 17:33:19 +0800 Subject: [PATCH 08/18] =?UTF-8?q?1.=20=E8=B0=83=E6=95=B4=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=A0=BC=E5=BC=8FLF=202.=20=E8=B0=83=E6=95=B4docker-compose?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=EF=BC=8C=E5=AE=B9=E5=99=A8=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 26 +++++++++---------- .../docker/knowstreaming-manage/Dockerfile | 6 ++--- .../docker/knowstreaming-manage/init/start.sh | 8 +----- km-dist/docker/knowstreaming-web/Dockerfile | 16 ++++++------ 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index 1e736d31..a687fe07 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -4,26 +4,26 @@ services: knowstreaming-manager: image: knowstreaming-manager:0.1.0 - container_name: knowstreaming-manager + container_name: knowstreaming_manager privileged: true restart: always depends_on: - - elasticsearch_single - - knowstreaming_mysql + - elasticsearch-single + - knowstreaming-mysql links: - - elasticsearch_single - - knowstreaming_mysql + - elasticsearch-single + - knowstreaming-mysql expose: - 80 environment: TZ: Asia/Shanghai - SERVER_MYSQL_ADDRESS: knowstreaming_mysql:3306 + SERVER_MYSQL_ADDRESS: knowstreaming-mysql:3306 SERVER_MYSQL_DB: know_streaming SERVER_MYSQL_USER: root SERVER_MYSQL_PASSWORD: admin2022_ - SERVER_ES_ADDRESS: elasticsearch_single:9200 + SERVER_ES_ADDRESS: elasticsearch-single:9200 JAVA_OPTS: -Xmx1g -Xms1g networks: @@ -32,7 +32,7 @@ services: # - /Users/haoqi/Downloads/ks/manage/log:/logs knowstreaming-ui: image: knowstreaming-ui:0.1.0 - container_name: knowstreaming-ui + container_name: knowstreaming_ui restart: always ports: - '80:80' @@ -46,7 +46,7 @@ services: - ks - elasticsearch_single: + elasticsearch-single: image: docker.io/library/elasticsearch:7.6.2 container_name: elasticsearch_single restart: always @@ -64,20 +64,20 @@ services: # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data networks: - ks - knowstreaming_init: + knowstreaming-init: image: knowstreaming-init:0.1.0 container_name: knowstreaming_init depends_on: - - elasticsearch_single + - elasticsearch-single links: - - elasticsearch_single + - elasticsearch-single environment: TZ: Asia/Shanghai SERVER_ES_ADDRESS: elasticsearch_single:9200 networks: - ks - knowstreaming_mysql: + knowstreaming-mysql: image: knowstreaming-mysql:0.1.0 container_name: knowstreaming_mysql restart: always diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index 95c20f7a..94553bce 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -5,7 +5,7 @@ LABEL maintainer="haoqi123 <1148648445@qq.com>" WORKDIR /knowstreaming ARG VERSION_MAVEN=3.6.3 -ARG VERSION_KM=v3.0.0-beta.2 +ARG VERSION_KM=3.0.0-beta.2 ARG INIT_FILE=./init/* ARG CONFIG_FILE=./conf/* @@ -24,10 +24,10 @@ RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_M # && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \ && git clone https://github.com/didi/KnowStreaming.git \ # && unzip -o -q master.zip \ - && cd KnowStreaming && git checkout ${VERSION_KM} \ + && cd KnowStreaming && git checkout v${VERSION_KM} \ && sed -i "/km-console/d" ./pom.xml \ && mvn -U clean package -Dmaven.test.skip=true \ - && cd / \ + && cd / && mkdir /app \ && mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \ && rm -rf /knowstreaming/* /usr/local/bin/mvn \ && yum erase -y git && yum clean all \ diff --git a/km-dist/docker/knowstreaming-manage/init/start.sh b/km-dist/docker/knowstreaming-manage/init/start.sh index 9741d7cf..79c5d77a 100644 --- a/km-dist/docker/knowstreaming-manage/init/start.sh +++ b/km-dist/docker/knowstreaming-manage/init/start.sh @@ -4,10 +4,4 @@ sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml -java -server ${JAVA_OPTS} \ --XX:+UseStringDeduplication \ --Dfile.encoding=UTF-8 \ --Djava.security.egd=file:/dev/./urandom \ --Duser.timezone=GMT+08 \ --XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log \ --jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file +java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-web/Dockerfile b/km-dist/docker/knowstreaming-web/Dockerfile index ef7e6b78..54b277db 100644 --- a/km-dist/docker/knowstreaming-web/Dockerfile +++ b/km-dist/docker/knowstreaming-web/Dockerfile @@ -4,8 +4,8 @@ LABEL maintainer="haoqi123 <1148648445@qq.com>" WORKDIR /knowstreaming -ARG VERSION_NODE=v14.20.0 -ARG VERSION_KM=v3.0.0-beta.2 +ARG VERSION_NODE=14.20.0 +ARG VERSION_KM=3.0.0-beta.2 # SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -59,9 +59,9 @@ RUN echo 'server { \ } \ }' > /etc/nginx/conf.d/default.conf -RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/${VERSION_NODE}/node-${VERSION_NODE}-linux-x64.tar.gz" \ +RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/v${VERSION_NODE}/node-v${VERSION_NODE}-linux-x64.tar.gz" \ && mkdir node/ \ - && tar -zxf "node-${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \ + && tar -zxf "node-v${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \ && rm -rf /knowstreaming/*.tar.gz \ && ln -s /knowstreaming/node/bin/node /usr/local/bin/node \ && ln -s /knowstreaming/node/bin/npm /usr/local/bin/npm \ @@ -70,11 +70,11 @@ RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/${VERSION_NODE && npm info underscore \ && node --version \ && npm --version \ - && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \ - && unzip -o -q master.zip \ - && cd KnowStreaming-master/km-console \ + && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/v${VERSION_KM}.zip \ + && unzip -o -q v${VERSION_KM}.zip \ + && cd KnowStreaming-${VERSION_KM}/km-console \ && npm install -g lerna && npm run i && npm run build \ - && mv /knowstreaming/KnowStreaming-master/km-rest/src/main/resources/templates /pub \ + && mv /knowstreaming/KnowStreaming-${VERSION_KM}/km-rest/src/main/resources/templates /pub \ && npm uninstall -g lerna \ && cd / \ && rm -rf /knowstreaming/* /usr/local/bin/node /usr/local/bin/npm \ From c73a7eee2f84aaba606e285a50a781938ff9b49b Mon Sep 17 00:00:00 2001 From: _haoqi <1148648445@qq.com> Date: Fri, 16 Sep 2022 20:03:58 +0800 Subject: [PATCH 09/18] =?UTF-8?q?1.=20=E8=B0=83=E6=95=B4docker-compose?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=EF=BC=8C=E5=AE=B9=E5=99=A8=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index a687fe07..b05e29b4 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -3,8 +3,8 @@ version: "3" services: knowstreaming-manager: - image: knowstreaming-manager:0.1.0 - container_name: knowstreaming_manager + image: haiti/knowstreaming-manager:0.1.0 + container_name: knowstreaming-manager privileged: true restart: always depends_on: @@ -28,14 +28,16 @@ services: JAVA_OPTS: -Xmx1g -Xms1g networks: - ks +# extra_hosts: +# - "hostname:x.x.x.x" # volumes: # - /Users/haoqi/Downloads/ks/manage/log:/logs knowstreaming-ui: - image: knowstreaming-ui:0.1.0 - container_name: knowstreaming_ui + image: haiti/knowstreaming-ui:0.1.0 + container_name: knowstreaming-ui restart: always ports: - - '80:80' + - '18092:80' environment: TZ: Asia/Shanghai depends_on: @@ -44,18 +46,19 @@ services: - knowstreaming-manager networks: - ks - +# extra_hosts: +# - "hostname:x.x.x.x" elasticsearch-single: image: docker.io/library/elasticsearch:7.6.2 - container_name: elasticsearch_single + container_name: elasticsearch-single restart: always expose: - 9200 - 9300 - ports: - - '9200:9200' - - '9300:9300' +# ports: +# - '9200:9200' +# - '9300:9300' environment: TZ: Asia/Shanghai ES_JAVA_OPTS: -Xms512m -Xmx512m @@ -65,7 +68,7 @@ services: networks: - ks knowstreaming-init: - image: knowstreaming-init:0.1.0 + image: haiti/knowstreaming-init:0.1.0 container_name: knowstreaming_init depends_on: - elasticsearch-single @@ -73,13 +76,13 @@ services: - elasticsearch-single environment: TZ: Asia/Shanghai - SERVER_ES_ADDRESS: elasticsearch_single:9200 + SERVER_ES_ADDRESS: elasticsearch-single:9200 networks: - ks knowstreaming-mysql: - image: knowstreaming-mysql:0.1.0 - container_name: knowstreaming_mysql + image: haiti/knowstreaming-mysql:0.1.0 + container_name: knowstreaming-mysql restart: always environment: TZ: Asia/Shanghai From a7a5fa9a31159a973f17038aa6695c8a132000a1 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Sun, 18 Sep 2022 19:10:22 +0800 Subject: [PATCH 10/18] =?UTF-8?q?1.=20=E8=B0=83=E6=95=B4docker-compose.yml?= =?UTF-8?q?=E4=B8=ADnetworks=E9=85=8D=E7=BD=AE=202.=20ks-manager=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=81=A5=E5=BA=B7=E6=A3=80=E6=9F=A5=203.=20=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=8D=95=E6=9C=BA=E9=83=A8=E7=BD=B2=E6=89=8B=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 90 +++++++++++++++++++ km-dist/docker/docker-compose.yml | 26 +----- .../docker/knowstreaming-manage/Dockerfile | 2 + 3 files changed, 96 insertions(+), 22 deletions(-) diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index f9f5ad1a..5110fa72 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -59,6 +59,8 @@ sh deploy_KnowStreaming-offline.sh ### 2.1.3、容器部署 +#### 2.1.3.1、Helm + **环境依赖** - Kubernetes >= 1.14 ，Helm >= 2.17.0 @@ -87,6 +89,94 @@ helm pull knowstreaming/knowstreaming-manager   +#### 2.1.3.2、Docker Compose +```yml +version: "3" + +services: + + knowstreaming-manager: + image: haiti/knowstreaming-manager:0.1.0 + container_name: knowstreaming-manager + privileged: true + restart: always + depends_on: + - elasticsearch-single + - knowstreaming-mysql + expose: + - 80 + environment: + TZ: Asia/Shanghai + + SERVER_MYSQL_ADDRESS: knowstreaming-mysql:3306 + SERVER_MYSQL_DB: know_streaming + SERVER_MYSQL_USER: root + SERVER_MYSQL_PASSWORD: admin2022_ + + SERVER_ES_ADDRESS: elasticsearch-single:9200 + + JAVA_OPTS: -Xmx1g -Xms1g + + # extra_hosts: + # - "hostname:x.x.x.x" + # volumes: + # - /Users/haoqi/Downloads/ks/manage/log:/logs + knowstreaming-ui: + image: haiti/knowstreaming-ui:0.1.0 + container_name: knowstreaming-ui + restart: always + ports: + - '18092:80' + environment: + TZ: Asia/Shanghai + depends_on: + - knowstreaming-manager + # extra_hosts: + # - "hostname:x.x.x.x" + + elasticsearch-single: + image: docker.io/library/elasticsearch:7.6.2 + container_name: elasticsearch-single + restart: always + expose: + - 9200 + - 9300 + # ports: + # - '9200:9200' + # - '9300:9300' + environment: + TZ: Asia/Shanghai + ES_JAVA_OPTS: -Xms512m -Xmx512m + discovery.type: single-node + # volumes: + # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data + + knowstreaming-init: + image: haiti/knowstreaming-init:0.1.0 + container_name: knowstreaming_init + depends_on: + - elasticsearch-single + environment: + TZ: Asia/Shanghai + SERVER_ES_ADDRESS: elasticsearch-single:9200 + + + knowstreaming-mysql: + image: haiti/knowstreaming-mysql:0.1.0 + container_name: knowstreaming-mysql + restart: always + environment: + TZ: Asia/Shanghai + MYSQL_ROOT_PASSWORD: admin2022_ + MYSQL_DATABASE: know_streaming + expose: + - 3306 +# volumes: +# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql +``` + +  + ### 2.1.4、手动部署 **部署流程** diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index b05e29b4..d916a51e 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -10,9 +10,6 @@ services: depends_on: - elasticsearch-single - knowstreaming-mysql - links: - - elasticsearch-single - - knowstreaming-mysql expose: - 80 environment: @@ -26,8 +23,7 @@ services: SERVER_ES_ADDRESS: elasticsearch-single:9200 JAVA_OPTS: -Xmx1g -Xms1g - networks: - - ks + # extra_hosts: # - "hostname:x.x.x.x" # volumes: @@ -42,10 +38,6 @@ services: TZ: Asia/Shanghai depends_on: - knowstreaming-manager - links: - - knowstreaming-manager - networks: - - ks # extra_hosts: # - "hostname:x.x.x.x" @@ -65,20 +57,16 @@ services: discovery.type: single-node # volumes: # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data - networks: - - ks + knowstreaming-init: image: haiti/knowstreaming-init:0.1.0 container_name: knowstreaming_init depends_on: - elasticsearch-single - links: - - elasticsearch-single environment: TZ: Asia/Shanghai SERVER_ES_ADDRESS: elasticsearch-single:9200 - networks: - - ks + knowstreaming-mysql: image: haiti/knowstreaming-mysql:0.1.0 @@ -88,13 +76,7 @@ services: TZ: Asia/Shanghai MYSQL_ROOT_PASSWORD: admin2022_ MYSQL_DATABASE: know_streaming - networks: - - ks expose: - 3306 # volumes: -# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql - -networks: - ks: - driver: bridge \ No newline at end of file +# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index 94553bce..b522eb5c 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -34,3 +34,5 @@ RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_M && rm -rf /root/.m2 /tmp/* ENTRYPOINT ["/bin/bash","/start.sh"] + +HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 From beb754aaaaef3950ee36c033f6024e7315e850c1 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 20 Sep 2022 14:22:06 +0800 Subject: [PATCH 11/18] =?UTF-8?q?=E4=BF=AE=E5=A4=8DJMX=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E8=A2=AB=E5=85=B3=E9=97=AD=EF=BC=8C=E6=8A=9B=E5=87=BAIOExcepti?= =?UTF-8?q?on=E5=90=8E=EF=BC=8C=E6=9C=AA=E8=BF=9B=E8=A1=8C=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E9=87=8D=E5=BB=BA=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../km/common/jmx/JmxConnectorWrap.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxConnectorWrap.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxConnectorWrap.java index 0fb65589..ca7c01c4 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxConnectorWrap.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxConnectorWrap.java @@ -90,6 +90,8 @@ public class JmxConnectorWrap { } try { jmxConnector.close(); + + jmxConnector = null; } catch (IOException e) { LOGGER.warn("close JmxConnector exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e); } @@ -105,6 +107,11 @@ public class JmxConnectorWrap { acquire(); MBeanServerConnection mBeanServerConnection = jmxConnector.getMBeanServerConnection(); return mBeanServerConnection.getAttribute(name, attribute); + } catch (IOException ioe) { + // 如果是因为连接断开，则进行重新连接，并抛出异常 + reInitDueIOException(); + + throw ioe; } finally { atomicInteger.incrementAndGet(); } @@ -120,6 +127,11 @@ public class JmxConnectorWrap { acquire(); MBeanServerConnection mBeanServerConnection = jmxConnector.getMBeanServerConnection(); return mBeanServerConnection.getAttributes(name, attributes); + } catch (IOException ioe) { + // 如果是因为连接断开，则进行重新连接，并抛出异常 + reInitDueIOException(); + + throw ioe; } finally { atomicInteger.incrementAndGet(); } @@ -131,6 +143,11 @@ public class JmxConnectorWrap { acquire(); MBeanServerConnection mBeanServerConnection = jmxConnector.getMBeanServerConnection(); return mBeanServerConnection.queryNames(name, query); + } catch (IOException ioe) { + // 如果是因为连接断开，则进行重新连接，并抛出异常 + reInitDueIOException(); + + throw ioe; } finally { atomicInteger.incrementAndGet(); } @@ -186,4 +203,26 @@ public class JmxConnectorWrap { } } } + + private synchronized void reInitDueIOException() { + try { + if (jmxConnector == null) { + return; + } + + // 检查是否正常 + jmxConnector.getConnectionId(); + + // 如果正常则直接返回 + return; + } catch (Exception e) { + // ignore + } + + // 关闭旧的 + this.close(); + + // 重新创建 + this.checkJmxConnectionAndInitIfNeed(); + } } From 0a03f41a7cadc446e67c9bb70725701dd3666d6c Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 20 Sep 2022 14:42:22 +0800 Subject: [PATCH 12/18] =?UTF-8?q?=E5=90=8E=E7=AB=AF=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E6=91=86=E6=94=BE=E9=A1=BA=E5=BA=8F=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/VersionControlManagerImpl.java | 36 +++++++++++++++++-- .../bean/dto/metrices/MetricDetailDTO.java | 28 +++++++++++++++ .../dto/metrices/UserMetricConfigDTO.java | 4 +++ .../config/metric/UserMetricConfig.java | 18 +++++++++- .../vo/config/metric/UserMetricConfigVO.java | 3 ++ .../km/common/constant/Constant.java | 2 ++ 6 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/MetricDetailDTO.java diff --git a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java index 994b2c8a..ad98d51a 100644 --- a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java +++ b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java @@ -7,12 +7,14 @@ import com.didiglobal.logi.log.LogFactory; import com.didiglobal.logi.security.common.dto.config.ConfigDTO; import com.didiglobal.logi.security.service.ConfigService; import com.xiaojukeji.know.streaming.km.biz.version.VersionControlManager; +import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDetailDTO; import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.UserMetricConfigDTO; import com.xiaojukeji.know.streaming.km.common.bean.entity.config.metric.UserMetricConfig; import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result; import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem; import com.xiaojukeji.know.streaming.km.common.bean.vo.config.metric.UserMetricConfigVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.version.VersionItemVO; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum; import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; import com.xiaojukeji.know.streaming.km.common.utils.VersionUtil; @@ -159,6 +161,9 @@ public class VersionControlManagerImpl implements VersionControlManager { UserMetricConfig umc = userMetricConfigMap.get(itemType + "@" + metric); userMetricConfigVO.setSet(null != umc && umc.isSet()); + if (umc != null) { + userMetricConfigVO.setRank(umc.getRank()); + } userMetricConfigVO.setName(itemVO.getName()); userMetricConfigVO.setType(itemVO.getType()); userMetricConfigVO.setDesc(itemVO.getDesc()); @@ -178,13 +183,30 @@ public class VersionControlManagerImpl implements VersionControlManager { @Override public Result updateUserMetricItem(Long clusterId, Integer type, UserMetricConfigDTO dto, String operator) { Map metricsSetMap = dto.getMetricsSet(); - if(null == metricsSetMap || metricsSetMap.isEmpty()){ + + //转换metricDetailDTOList + List metricDetailDTOList = dto.getMetricDetailDTOList(); + Map metricDetailMap = new HashMap<>(); + if (metricDetailDTOList != null && !metricDetailDTOList.isEmpty()) { + metricDetailMap = metricDetailDTOList.stream().collect(Collectors.toMap(MetricDetailDTO::getMetric, Function.identity())); + } + + //转换metricsSetMap + if (metricsSetMap != null && !metricsSetMap.isEmpty()) { + for (Map.Entry metricAndShowEntry : metricsSetMap.entrySet()) { + if (metricDetailMap.containsKey(metricAndShowEntry.getKey())) continue; + metricDetailMap.put(metricAndShowEntry.getKey(), new MetricDetailDTO(metricAndShowEntry.getKey(), metricAndShowEntry.getValue(), null)); + } + } + + if (metricDetailMap.isEmpty()) { return Result.buildSuc(); } + Set userMetricConfigs = getUserMetricConfig(operator); for(Map.Entry metricAndShowEntry : metricsSetMap.entrySet()){ - UserMetricConfig userMetricConfig = new UserMetricConfig(type, metricAndShowEntry.getKey(), metricAndShowEntry.getValue()); + UserMetricConfig userMetricConfig = new UserMetricConfig(type, metricAndShowEntry.getKey(), metricAndShowEntry.getValue(), metricDetailMap.get(metricAndShowEntry.getKey()).getRank()); userMetricConfigs.remove(userMetricConfig); userMetricConfigs.add(userMetricConfig); } @@ -228,7 +250,15 @@ public class VersionControlManagerImpl implements VersionControlManager { return defaultMetrics; } - return JSON.parseObject(value, new TypeReference>(){}); + Set userMetricConfigs = JSON.parseObject(value, new TypeReference>() {}); + + //补充rank不存在情况 + for (UserMetricConfig userMetricConfig : userMetricConfigs) { + if (userMetricConfig.getRank() == 0) { + userMetricConfig.setRank(Constant.DEFAULT_METRIC_RANK); + } + } + return userMetricConfigs; } public static void main(String[] args){ diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/MetricDetailDTO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/MetricDetailDTO.java new file mode 100644 index 00000000..0d51d2c6 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/MetricDetailDTO.java @@ -0,0 +1,28 @@ +package com.xiaojukeji.know.streaming.km.common.bean.dto.metrices; + +import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author didi + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(description = "指标详细属性信息") +public class MetricDetailDTO extends BaseDTO { + + @ApiModelProperty("指标名称") + private String metric; + + @ApiModelProperty("指标是否显示") + private Boolean set; + + @ApiModelProperty("指标优先级") + private Integer rank; + +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/UserMetricConfigDTO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/UserMetricConfigDTO.java index 02bb1d2a..bf71b663 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/UserMetricConfigDTO.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/UserMetricConfigDTO.java @@ -7,6 +7,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; import java.util.Map; @@ -17,4 +18,7 @@ import java.util.Map; public class UserMetricConfigDTO extends BaseDTO { @ApiModelProperty("指标展示设置项，key：指标名；value：是否展现(true展现/false不展现)") private Map metricsSet; + + @ApiModelProperty("指标自定义属性列表") + private List metricDetailDTOList; } diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java index 6895fb40..9f85d7ca 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java @@ -1,12 +1,12 @@ package com.xiaojukeji.know.streaming.km.common.bean.entity.config.metric; +import com.xiaojukeji.know.streaming.km.common.constant.Constant; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data @NoArgsConstructor -@AllArgsConstructor public class UserMetricConfig { private int type; @@ -15,6 +15,22 @@ public class UserMetricConfig { private boolean set; + private int rank; + + public UserMetricConfig(int type, String metric, boolean set, Integer rank) { + this.type = type; + this.metric = metric; + this.set = set; + this.rank = rank == null ? Constant.DEFAULT_METRIC_RANK : rank; + } + + public UserMetricConfig(int type, String metric, boolean set) { + this.type = type; + this.metric = metric; + this.set = set; + this.rank = Constant.DEFAULT_METRIC_RANK; + } + @Override public int hashCode(){ return metric.hashCode() << 1 + type; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/config/metric/UserMetricConfigVO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/config/metric/UserMetricConfigVO.java index e50fc6e7..2b4e76b3 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/config/metric/UserMetricConfigVO.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/config/metric/UserMetricConfigVO.java @@ -14,4 +14,7 @@ import lombok.NoArgsConstructor; public class UserMetricConfigVO extends VersionItemVO { @ApiModelProperty(value = "该指标用户是否设置展现", example = "true") private Boolean set; + + @ApiModelProperty(value = "该指标展示优先级", example = "1") + private Integer rank; } diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java index 36575938..f86c1d8c 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java @@ -42,6 +42,8 @@ public class Constant { */ public static final Integer DEFAULT_CLUSTER_HEALTH_SCORE = 90; + public static final Integer DEFAULT_METRIC_RANK = 10000; + public static final String DEFAULT_USER_NAME = "know-streaming-app"; public static final int INVALID_CODE = -1; From 9a25c22f3a8934c929be4ad3740d0296add98759 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Tue, 20 Sep 2022 19:23:18 +0800 Subject: [PATCH 13/18] =?UTF-8?q?1.=20=E8=B0=83=E6=95=B4docker-compose.yml?= =?UTF-8?q?=E4=B8=AD=E5=90=84=E4=B8=AA=E6=9C=8D=E5=8A=A1=E7=9A=84=E9=95=9C?= =?UTF-8?q?=E5=83=8F=202.=20=E7=BB=8F=E8=BF=87@wangdongfang-aden=E5=A4=A7?= =?UTF-8?q?=E5=93=A5=E7=9A=84=E8=B0=83=E8=AF=95=E5=B0=86helm=E4=B8=8Edocke?= =?UTF-8?q?r=E9=95=9C=E5=83=8F=E5=90=88=E4=BA=8C=E4=B8=BA=E4=B8=80?= =?UTF-8?q?=EF=BC=8C=E4=BA=8E=E6=98=AF=E5=88=A0=E5=87=8F=E6=8E=89=E5=90=84?= =?UTF-8?q?=E4=B8=AA=E9=95=9C=E5=83=8F=E7=9A=84Dockerfile=E4=B8=8E?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E8=84=9A=E6=9C=AC=EF=BC=8C=E5=90=8E=E7=BB=AD?= =?UTF-8?q?=E4=B9=9F=E4=B8=8D=E9=9C=80=E8=A6=81=E9=A2=9D=E5=A4=96=E7=BB=B4?= =?UTF-8?q?=E6=8A=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- km-dist/docker/docker-compose.yml | 23 +- km-dist/docker/knowstreaming-init/Dockerfile | 10 - .../knowstreaming-init/init/template.sh | 661 ------------------ .../docker/knowstreaming-manage/Dockerfile | 38 - .../knowstreaming-manage/conf/application.yml | 107 --- .../docker/knowstreaming-manage/init/start.sh | 7 - km-dist/docker/knowstreaming-mysql/Dockerfile | 10 - .../docker/knowstreaming-mysql/conf/mysql.cnf | 15 - km-dist/docker/knowstreaming-web/Dockerfile | 86 --- 9 files changed, 16 insertions(+), 941 deletions(-) delete mode 100644 km-dist/docker/knowstreaming-init/Dockerfile delete mode 100644 km-dist/docker/knowstreaming-init/init/template.sh delete mode 100644 km-dist/docker/knowstreaming-manage/Dockerfile delete mode 100644 km-dist/docker/knowstreaming-manage/conf/application.yml delete mode 100644 km-dist/docker/knowstreaming-manage/init/start.sh delete mode 100644 km-dist/docker/knowstreaming-mysql/Dockerfile delete mode 100644 km-dist/docker/knowstreaming-mysql/conf/mysql.cnf delete mode 100644 km-dist/docker/knowstreaming-web/Dockerfile diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index d916a51e..a9fe2bf3 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: knowstreaming-manager: - image: haiti/knowstreaming-manager:0.1.0 + image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming-manager privileged: true restart: always @@ -12,6 +12,9 @@ services: - knowstreaming-mysql expose: - 80 + command: + - /bin/sh + - /ks-start.sh environment: TZ: Asia/Shanghai @@ -27,9 +30,9 @@ services: # extra_hosts: # - "hostname:x.x.x.x" # volumes: -# - /Users/haoqi/Downloads/ks/manage/log:/logs +# - /ks/manage/log:/logs knowstreaming-ui: - image: haiti/knowstreaming-ui:0.1.0 + image: knowstreaming/knowstreaming-ui:0.2.0-test1 container_name: knowstreaming-ui restart: always ports: @@ -56,27 +59,33 @@ services: ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node # volumes: -# - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data +# - /ks/es/data:/usr/share/elasticsearch/data knowstreaming-init: - image: haiti/knowstreaming-init:0.1.0 + image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming_init depends_on: - elasticsearch-single + command: + - /bin/bash + - /es_template_create.sh environment: TZ: Asia/Shanghai SERVER_ES_ADDRESS: elasticsearch-single:9200 knowstreaming-mysql: - image: haiti/knowstreaming-mysql:0.1.0 + image: knowstreaming/knowstreaming-mysql:0.2.0-test container_name: knowstreaming-mysql restart: always environment: TZ: Asia/Shanghai MYSQL_ROOT_PASSWORD: admin2022_ MYSQL_DATABASE: know_streaming + MYSQL_ROOT_HOST: '%' expose: - 3306 +# ports: +# - '3306:3306' # volumes: -# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql \ No newline at end of file +# - /ks/mysql/data:/data/mysql \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/Dockerfile b/km-dist/docker/knowstreaming-init/Dockerfile deleted file mode 100644 index ddb2120c..00000000 --- a/km-dist/docker/knowstreaming-init/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM debian:stable-20220912-slim -LABEL org.opencontainers.image.authors="haoqi123" -ARG INIT_ES_FILE=./init/* -COPY ${INIT_ES_FILE} / -ENV TZ=Asia/Shanghai - -RUN apt-get update \ - && apt-get install -y curl - -ENTRYPOINT ["/bin/bash","/template.sh","start"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/template.sh b/km-dist/docker/knowstreaming-init/init/template.sh deleted file mode 100644 index ce41db73..00000000 --- a/km-dist/docker/knowstreaming-init/init/template.sh +++ /dev/null @@ -1,661 +0,0 @@ -echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}" -while true -do - curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 - if [ "$?" != "0" ];then - sleep 1s - else - echo "ElasticSearch Start Initialize" - break - fi -done - -curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_broker_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "NetworkProcessorAvgIdle" : { - "type" : "float" - }, - "UnderReplicatedPartitions" : { - "type" : "float" - }, - "BytesIn_min_15" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "RequestHandlerAvgIdle" : { - "type" : "float" - }, - "connectionsCount" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "TotalRequestQueueSize" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "TotalResponseQueueSize" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "Connections" : { - "type" : "double" - }, - "BytesIn_min_15" : { - "type" : "double" - }, - "PartitionURP" : { - "type" : "double" - }, - "HealthScore_Topics" : { - "type" : "double" - }, - "EventQueueSize" : { - "type" : "double" - }, - "ActiveControllerCount" : { - "type" : "double" - }, - "GroupDeads" : { - "type" : "double" - }, - "BytesIn_min_5" : { - "type" : "double" - }, - "HealthCheckTotal_Topics" : { - "type" : "double" - }, - "Partitions" : { - "type" : "double" - }, - "BytesOut" : { - "type" : "double" - }, - "Groups" : { - "type" : "double" - }, - "BytesOut_min_15" : { - "type" : "double" - }, - "TotalRequestQueueSize" : { - "type" : "double" - }, - "HealthCheckPassed_Groups" : { - "type" : "double" - }, - "TotalProduceRequests" : { - "type" : "double" - }, - "HealthCheckPassed" : { - "type" : "double" - }, - "TotalLogSize" : { - "type" : "double" - }, - "GroupEmptys" : { - "type" : "double" - }, - "PartitionNoLeader" : { - "type" : "double" - }, - "HealthScore_Brokers" : { - "type" : "double" - }, - "Messages" : { - "type" : "double" - }, - "Topics" : { - "type" : "double" - }, - "PartitionMinISR_E" : { - "type" : "double" - }, - "HealthCheckTotal" : { - "type" : "double" - }, - "Brokers" : { - "type" : "double" - }, - "Replicas" : { - "type" : "double" - }, - "HealthCheckTotal_Groups" : { - "type" : "double" - }, - "GroupRebalances" : { - "type" : "double" - }, - "MessageIn" : { - "type" : "double" - }, - "HealthScore" : { - "type" : "double" - }, - "HealthCheckPassed_Topics" : { - "type" : "double" - }, - "HealthCheckTotal_Brokers" : { - "type" : "double" - }, - "PartitionMinISR_S" : { - "type" : "double" - }, - "BytesIn" : { - "type" : "double" - }, - "BytesOut_min_5" : { - "type" : "double" - }, - "GroupActives" : { - "type" : "double" - }, - "MessagesIn" : { - "type" : "double" - }, - "GroupReBalances" : { - "type" : "double" - }, - "HealthCheckPassed_Brokers" : { - "type" : "double" - }, - "HealthScore_Groups" : { - "type" : "double" - }, - "TotalResponseQueueSize" : { - "type" : "double" - }, - "Zookeepers" : { - "type" : "double" - }, - "LeaderMessages" : { - "type" : "double" - }, - "HealthScore_Cluster" : { - "type" : "double" - }, - "HealthCheckPassed_Cluster" : { - "type" : "double" - }, - "HealthCheckTotal_Cluster" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_group_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "group" : { - "type" : "keyword" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "HealthScore" : { - "type" : "float" - }, - "Lag" : { - "type" : "float" - }, - "OffsetConsumed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "groupMetric" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }[root@10-255-0-23 template]# cat ks_kafka_replication_metric -PUT _template/ks_kafka_replication_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_replication_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_topic_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "topic" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "BytesIn_min_15" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "BytesRejected" : { - "type" : "float" - }, - "PartitionURP" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ReplicationCount" : { - "type" : "float" - }, - "ReplicationBytesOut" : { - "type" : "float" - }, - "ReplicationBytesIn" : { - "type" : "float" - }, - "FailedFetchRequests" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "LogSize" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "FailedProduceRequests" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "brokerAgg" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -for i in {0..6}; -do - logdate=_$(date -d "${i} day ago" +%Y-%m-%d) - curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ - exit 2 -done - -echo "ElasticSearch Initialize Success" \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile deleted file mode 100644 index b522eb5c..00000000 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM docker.io/amd64/eclipse-temurin:17.0.4.1_1-jdk-centos7 - -LABEL maintainer="haoqi123 <1148648445@qq.com>" - -WORKDIR /knowstreaming - -ARG VERSION_MAVEN=3.6.3 -ARG VERSION_KM=3.0.0-beta.2 -ARG INIT_FILE=./init/* -ARG CONFIG_FILE=./conf/* - -COPY ${INIT_FILE} / -COPY ${CONFIG_FILE} /conf/ - -# SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN yum install -y git - -RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_MAVEN}/binaries/apache-maven-${VERSION_MAVEN}-bin.tar.gz" \ - && mkdir maven/ \ - && tar -zxf "apache-maven-${VERSION_MAVEN}-bin.tar.gz" -C maven/ --strip-components=1 --no-same-owner \ - && rm -rf /knowstreaming/*.tar.gz \ - && ln -s /knowstreaming/maven/bin/mvn /usr/local/bin/mvn \ -# && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \ - && git clone https://github.com/didi/KnowStreaming.git \ -# && unzip -o -q master.zip \ - && cd KnowStreaming && git checkout v${VERSION_KM} \ - && sed -i "/km-console/d" ./pom.xml \ - && mvn -U clean package -Dmaven.test.skip=true \ - && cd / && mkdir /app \ - && mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \ - && rm -rf /knowstreaming/* /usr/local/bin/mvn \ - && yum erase -y git && yum clean all \ - && rm -rf /root/.m2 /tmp/* - -ENTRYPOINT ["/bin/bash","/start.sh"] - -HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 diff --git a/km-dist/docker/knowstreaming-manage/conf/application.yml b/km-dist/docker/knowstreaming-manage/conf/application.yml deleted file mode 100644 index 80323cd0..00000000 --- a/km-dist/docker/knowstreaming-manage/conf/application.yml +++ /dev/null @@ -1,107 +0,0 @@ -server: - port: 80 # 服务端口 - tomcat: - accept-count: 1000 - max-connections: 10000 - -spring: - application: - name: know-streaming - profiles: - active: dev - main: - allow-bean-definition-overriding: true - jackson: - time-zone: GMT+8 - datasource: - know-streaming: # know-streaming 自身数据库的配置 - jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - username: SERVER_MYSQL_USER - password: SERVER_MYSQL_PASSWORD - driver-class-name: org.mariadb.jdbc.Driver - maximum-pool-size: 20 - idle-timeout: 30000 - connection-test-query: SELECT 1 - logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 - jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - username: SERVER_MYSQL_USER - password: SERVER_MYSQL_PASSWORD - driver-class-name: org.mariadb.jdbc.Driver - max-lifetime: 60000 - init-sql: true - init-thread-num: 20 - max-thread-num: 50 - log-expire: 3 # 日志保存天数，以天为单位 - app-name: know-streaming - claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual - logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 - jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - username: SERVER_MYSQL_USER - password: SERVER_MYSQL_PASSWORD - driver-class-name: org.mariadb.jdbc.Driver - app-name: know-streaming - resource-extend-bean-name: myResourceExtendImpl - login-extend-bean-name: logiSecurityDefaultLoginExtendImpl - -logging: - config: classpath:logback-spring.xml - -# 线程池大小相关配置 -thread-pool: - scheduled: - thread-num: 2 # @Scheduled任务的线程池大小，默认是一个 - - collector: # 采集模块的配置 - future-util: # 采集模块线程池配置 - num: 3 # 线程池个数 - thread-num: 64 # 每个线程池核心线程数 - queue-size: 10000 # 每个线程池队列大小 - select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改 - suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改 - - task: # 任务模块的配置 - metrics: # metrics采集任务配置 - thread-num: 18 # metrics采集任务线程池核心线程数 - queue-size: 180 # metrics采集任务线程池队列大小 - metadata: # metadata同步任务配置 - thread-num: 27 # metadata同步任务线程池核心线程数 - queue-size: 270 # metadata同步任务线程池队列大小 - common: # 剩余其他任务配置 - thread-num: 15 # 剩余其他任务线程池核心线程数 - queue-size: 150 # 剩余其他任务线程池队列大小 - - -# 客户端池大小相关配置 -client-pool: - kafka-consumer: - min-idle-client-num: 2 # 最小空闲客户端数 - max-idle-client-num: 20 # 最大空闲客户端数 - max-total-client-num: 20 # 最大客户端数 - borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒 - - -# ES客户端配置 -es: - client: - address: SERVER_ES_ADDRESS - client-cnt: 10 - io-thread-cnt: 2 - max-retry-cnt: 5 - -# 普罗米修斯指标导出相关配置 -management: - endpoints: - web: - base-path: /metrics - exposure: - include: '*' - health: - elasticsearch: - enabled: false - metrics: - export: - prometheus: - descriptions: true - enabled: true - tags: - application: know-streaming \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/init/start.sh b/km-dist/docker/knowstreaming-manage/init/start.sh deleted file mode 100644 index 79c5d77a..00000000 --- a/km-dist/docker/knowstreaming-manage/init/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml -sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml - -java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile deleted file mode 100644 index 95720f05..00000000 --- a/km-dist/docker/knowstreaming-mysql/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM mysql:5.7.39-oracle - -LABEL org.opencontainers.image.authors="haoqi123" - -EXPOSE 3306 - -ARG INIT_SOURCE_FILE=./sql/*.sql -ARG MYSQL_CNF_FILE=./conf/*.cnf -COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/ -COPY ${MYSQL_CNF_FILE} /etc/mysql/conf.d/ \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf b/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf deleted file mode 100644 index 37c97010..00000000 --- a/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf +++ /dev/null @@ -1,15 +0,0 @@ -[client] -default-character-set=utf8 -[mysql] -default-character-set=utf8 -[mysqld] -skip-host-cache -skip-name-resolve -datadir=/data/mysql -socket=/var/lib/mysql/mysql.sock -secure-file-priv=/var/lib/mysql-files -character-set-server=utf8 -user=mysql -symbolic-links=0 -pid-file=/var/run/mysqld/mysqld.pid -sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-web/Dockerfile b/km-dist/docker/knowstreaming-web/Dockerfile deleted file mode 100644 index 54b277db..00000000 --- a/km-dist/docker/knowstreaming-web/Dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -FROM docker.io/amd64/nginx:1.22 - -LABEL maintainer="haoqi123 <1148648445@qq.com>" - -WORKDIR /knowstreaming - -ARG VERSION_NODE=14.20.0 -ARG VERSION_KM=3.0.0-beta.2 - -# SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN apt update \ - && apt install -y git unzip - -RUN echo 'server { \ - listen 80; \ - server_name localhost; \ - gzip on; \ - gzip_buffers 16 8k; \ - gzip_comp_level 4; \ - gzip_http_version 1.0; \ - gzip_min_length 1280; \ - gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; \ - gzip_vary on; \ - root /pub; \ - location / { \ - root /pub; \ - if ($request_filename ~* .*\.(?:htm|html|json)$) { \ - add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; \ - } \ - try_files $uri /layout/index.html; \ - } \ - location ~* \.(json)$ { \ - add_header Cache-Control no-cache; \ - } \ - location @kmfallback { \ - } \ - #location ~ ^/(clusters|config|cluster|login) { \ - # rewrite ^.*$ /; \ - #} \ - location ~ ^/ks-km/api/v3 { \ - #rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; \ - proxy_pass http://knowstreaming-manager; \ - #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \ - #proxy_cookie_path /ks-km/ /; \ - #proxy_set_header Host $host; \ - #proxy_set_header Referer $http_referer; \ - #proxy_set_header Cookie $http_cookie; \ - #proxy_set_header X-Real-Ip $remote_addr; \ - #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; \ - } \ - location ~ ^/logi-security/api/v1 { \ - #rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; \ - proxy_pass http://knowstreaming-manager; \ - #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \ - } \ - location ~ ^/(401|403|404|500){ \ - rewrite ^.*$ /; \ - } \ - }' > /etc/nginx/conf.d/default.conf - -RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/v${VERSION_NODE}/node-v${VERSION_NODE}-linux-x64.tar.gz" \ - && mkdir node/ \ - && tar -zxf "node-v${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \ - && rm -rf /knowstreaming/*.tar.gz \ - && ln -s /knowstreaming/node/bin/node /usr/local/bin/node \ - && ln -s /knowstreaming/node/bin/npm /usr/local/bin/npm \ - && npm config set strict-ssl false \ - && npm config set registry https://registry.npm.taobao.org \ - && npm info underscore \ - && node --version \ - && npm --version \ - && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/v${VERSION_KM}.zip \ - && unzip -o -q v${VERSION_KM}.zip \ - && cd KnowStreaming-${VERSION_KM}/km-console \ - && npm install -g lerna && npm run i && npm run build \ - && mv /knowstreaming/KnowStreaming-${VERSION_KM}/km-rest/src/main/resources/templates /pub \ - && npm uninstall -g lerna \ - && cd / \ - && rm -rf /knowstreaming/* /usr/local/bin/node /usr/local/bin/npm \ - && apt remove -y git unzip && apt autoremove -y && apt remove --purge --auto-remove -y \ - && rm -rf /var/lib/apt/lists/* /tmp /root/.npm - -#ENTRYPOINT ["ls", "/ks_ui/km-console"] - -#CMD ["nginx", "-g", "daemon off;"] From e626b990904cb81436d522db23deb9ad6e00d1c4 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Tue, 20 Sep 2022 19:30:20 +0800 Subject: [PATCH 14/18] =?UTF-8?q?1.=20=E5=88=A0=E9=99=A4km-dist/docker?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9=EF=BC=8C=E4=BB=A5[=E5=8D=95?= =?UTF-8?q?=E6=9C=BA=E9=83=A8=E7=BD=B2=E6=89=8B=E5=86=8C.md]=E4=B8=BA?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 43 ++++++++------ km-dist/docker/docker-compose.yml | 91 ------------------------------ 2 files changed, 26 insertions(+), 108 deletions(-) delete mode 100644 km-dist/docker/docker-compose.yml diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index 5110fa72..b429b9d5 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -94,9 +94,9 @@ helm pull knowstreaming/knowstreaming-manager version: "3" services: - + knowstreaming-manager: - image: haiti/knowstreaming-manager:0.1.0 + image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming-manager privileged: true restart: always @@ -105,6 +105,9 @@ services: - knowstreaming-mysql expose: - 80 + command: + - /bin/sh + - /ks-start.sh environment: TZ: Asia/Shanghai @@ -117,12 +120,12 @@ services: JAVA_OPTS: -Xmx1g -Xms1g - # extra_hosts: - # - "hostname:x.x.x.x" - # volumes: - # - /Users/haoqi/Downloads/ks/manage/log:/logs +# extra_hosts: +# - "hostname:x.x.x.x" +# volumes: +# - /ks/manage/log:/logs knowstreaming-ui: - image: haiti/knowstreaming-ui:0.1.0 + image: knowstreaming/knowstreaming-ui:0.2.0-test1 container_name: knowstreaming-ui restart: always ports: @@ -131,8 +134,8 @@ services: TZ: Asia/Shanghai depends_on: - knowstreaming-manager - # extra_hosts: - # - "hostname:x.x.x.x" +# extra_hosts: +# - "hostname:x.x.x.x" elasticsearch-single: image: docker.io/library/elasticsearch:7.6.2 @@ -141,38 +144,44 @@ services: expose: - 9200 - 9300 - # ports: - # - '9200:9200' - # - '9300:9300' +# ports: +# - '9200:9200' +# - '9300:9300' environment: TZ: Asia/Shanghai ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node - # volumes: - # - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data +# volumes: +# - /ks/es/data:/usr/share/elasticsearch/data knowstreaming-init: - image: haiti/knowstreaming-init:0.1.0 + image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming_init depends_on: - elasticsearch-single + command: + - /bin/bash + - /es_template_create.sh environment: TZ: Asia/Shanghai SERVER_ES_ADDRESS: elasticsearch-single:9200 knowstreaming-mysql: - image: haiti/knowstreaming-mysql:0.1.0 + image: knowstreaming/knowstreaming-mysql:0.2.0-test container_name: knowstreaming-mysql restart: always environment: TZ: Asia/Shanghai MYSQL_ROOT_PASSWORD: admin2022_ MYSQL_DATABASE: know_streaming + MYSQL_ROOT_HOST: '%' expose: - 3306 +# ports: +# - '3306:3306' # volumes: -# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql +# - /ks/mysql/data:/data/mysql ```   diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml deleted file mode 100644 index a9fe2bf3..00000000 --- a/km-dist/docker/docker-compose.yml +++ /dev/null @@ -1,91 +0,0 @@ -version: "3" - -services: - - knowstreaming-manager: - image: knowstreaming/knowstreaming-manager:0.2.0-test - container_name: knowstreaming-manager - privileged: true - restart: always - depends_on: - - elasticsearch-single - - knowstreaming-mysql - expose: - - 80 - command: - - /bin/sh - - /ks-start.sh - environment: - TZ: Asia/Shanghai - - SERVER_MYSQL_ADDRESS: knowstreaming-mysql:3306 - SERVER_MYSQL_DB: know_streaming - SERVER_MYSQL_USER: root - SERVER_MYSQL_PASSWORD: admin2022_ - - SERVER_ES_ADDRESS: elasticsearch-single:9200 - - JAVA_OPTS: -Xmx1g -Xms1g - -# extra_hosts: -# - "hostname:x.x.x.x" -# volumes: -# - /ks/manage/log:/logs - knowstreaming-ui: - image: knowstreaming/knowstreaming-ui:0.2.0-test1 - container_name: knowstreaming-ui - restart: always - ports: - - '18092:80' - environment: - TZ: Asia/Shanghai - depends_on: - - knowstreaming-manager -# extra_hosts: -# - "hostname:x.x.x.x" - - elasticsearch-single: - image: docker.io/library/elasticsearch:7.6.2 - container_name: elasticsearch-single - restart: always - expose: - - 9200 - - 9300 -# ports: -# - '9200:9200' -# - '9300:9300' - environment: - TZ: Asia/Shanghai - ES_JAVA_OPTS: -Xms512m -Xmx512m - discovery.type: single-node -# volumes: -# - /ks/es/data:/usr/share/elasticsearch/data - - knowstreaming-init: - image: knowstreaming/knowstreaming-manager:0.2.0-test - container_name: knowstreaming_init - depends_on: - - elasticsearch-single - command: - - /bin/bash - - /es_template_create.sh - environment: - TZ: Asia/Shanghai - SERVER_ES_ADDRESS: elasticsearch-single:9200 - - - knowstreaming-mysql: - image: knowstreaming/knowstreaming-mysql:0.2.0-test - container_name: knowstreaming-mysql - restart: always - environment: - TZ: Asia/Shanghai - MYSQL_ROOT_PASSWORD: admin2022_ - MYSQL_DATABASE: know_streaming - MYSQL_ROOT_HOST: '%' - expose: - - 3306 -# ports: -# - '3306:3306' -# volumes: -# - /ks/mysql/data:/data/mysql \ No newline at end of file From 7da1e67b0173de61d976768ccae779c35ad7080d Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 22 Sep 2022 11:13:54 +0800 Subject: [PATCH 15/18] =?UTF-8?q?FAQ=E8=A1=A5=E5=85=85=E6=9D=83=E9=99=90?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user_guide/faq.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/user_guide/faq.md b/docs/user_guide/faq.md index 764c58b9..98dfbf83 100644 --- a/docs/user_guide/faq.md +++ b/docs/user_guide/faq.md @@ -166,3 +166,19 @@ Node 版本: v12.22.12 需要到具体的应用中执行 `npm run start`，例如 `cd packages/layout-clusters-fe` 后，执行 `npm run start`。 应用启动后需要到基座应用中查看（需要启动基座应用，即 layout-clusters-fe）。 + + +## 8.12、权限识别失败问题 +1、使用admin账号登陆KnowStreaming时，点击系统管理-用户管理-角色管理-新增角色，查看页面是否正常。 + + + +2、查看'/logi-security/api/v1/permission/tree'接口返回值，出现如下图所示乱码现象。 +![接口返回值](http://img-ys011.didistatic.com/static/dc2img/do1_jTxBkwNGU9vZuYQQbdNw) + +3、查看logi_security_permission表，看看是否出现了中文乱码现象。 + +根据以上几点，我们可以确定是由于数据库乱码造成的权限识别失败问题。 + ++ 原因：由于数据库编码和我们提供的脚本不一致，数据库里的数据发生了乱码，因此出现权限识别失败问题。 ++ 解决方案：清空数据库数据，将数据库字符集调整为utf8，最后重新执行[dml-logi.sql](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/sql/dml-logi.sql)脚本导入数据即可。 From 833c360698054a17591f49845d36ba639d06bf50 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 22 Sep 2022 11:17:59 +0800 Subject: [PATCH 16/18] bump oshi-core version to 5.6.1 --- km-task/pom.xml | 14 ++++++++++++++ pom.xml | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/km-task/pom.xml b/km-task/pom.xml index 502d806d..d07b37b3 100644 --- a/km-task/pom.xml +++ b/km-task/pom.xml @@ -43,7 +43,21 @@ io.github.zqrferrari logi-job-spring-boot-starter + + + oshi-core + com.github.oshi + + + + + + com.github.oshi + oshi-core + 5.6.1 + + io.github.zqrferrari logi-security-spring-boot-starter diff --git a/pom.xml b/pom.xml index e30add73..619f24c7 100644 --- a/pom.xml +++ b/pom.xml @@ -230,6 +230,19 @@ io.github.zqrferrari logi-job-spring-boot-starter 1.0.23 + + + oshi-core + com.github.oshi + + + + + + + com.github.oshi + oshi-core + 5.6.1 From 2b76358c8f210e6d87d855e94452824cc68a2bc8 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 22 Sep 2022 11:24:13 +0800 Subject: [PATCH 17/18] =?UTF-8?q?Overview=E9=A1=B5=E9=9D=A2=EF=BC=8C?= =?UTF-8?q?=E5=90=8E=E7=AB=AF=E5=A2=9E=E5=8A=A0=E6=8E=92=E5=BA=8F=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/VersionControlManagerImpl.java | 49 ++++++++----------- .../config/metric/UserMetricConfig.java | 6 +-- .../km/common/constant/Constant.java | 1 - 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java index ad98d51a..52a91520 100644 --- a/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java +++ b/km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java @@ -49,29 +49,29 @@ public class VersionControlManagerImpl implements VersionControlManager { @PostConstruct public void init(){ defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_HEALTH_SCORE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_TOTAL_PRODUCE_REQUESTS, true)); defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_FAILED_FETCH_REQ, true)); defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_FAILED_PRODUCE_REQ, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_MESSAGE_IN, true)); defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_UNDER_REPLICA_PARTITIONS, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_TOTAL_PRODUCE_REQUESTS, true)); defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_IN, true)); defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_OUT, true)); defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_BYTES_REJECTED, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_TOPIC.getCode(), TOPIC_METRIC_MESSAGE_IN, true)); defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_HEALTH_SCORE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_REQ_QUEUE_SIZE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_RES_QUEUE_SIZE, true)); defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_ACTIVE_CONTROLLER_COUNT, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_PRODUCE_REQ, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_LOG_SIZE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_CONNECTIONS, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_MESSAGES_IN, true)); defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_BYTES_IN, true)); defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_BYTES_OUT, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_GROUP_REBALANCES, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_JOB_RUNNING, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_CONNECTIONS, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_MESSAGES_IN, true)); defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_PARTITIONS_NO_LEADER, true)); defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_PARTITION_URP, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_LOG_SIZE, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_PRODUCE_REQ, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_REQ_QUEUE_SIZE, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_TOTAL_RES_QUEUE_SIZE, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_GROUP_REBALANCES, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_CLUSTER.getCode(), CLUSTER_METRIC_JOB_RUNNING, true)); defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_OFFSET_CONSUMED, true)); defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_LAG, true)); @@ -79,18 +79,18 @@ public class VersionControlManagerImpl implements VersionControlManager { defaultMetrics.add(new UserMetricConfig(METRIC_GROUP.getCode(), GROUP_METRIC_HEALTH_SCORE, true)); defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_HEALTH_SCORE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_REQ_QUEUE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_RES_QUEUE, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_CONNECTION_COUNT, true)); defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_MESSAGE_IN, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_PRODUCE_REQ, true)); defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_NETWORK_RPO_AVG_IDLE, true)); defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_REQ_AVG_IDLE, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_CONNECTION_COUNT, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_IN, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_OUT, true)); - defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_PARTITIONS_SKEW, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_PRODUCE_REQ, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_REQ_QUEUE, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_TOTAL_RES_QUEUE, true)); defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_LEADERS_SKEW, true)); defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_UNDER_REPLICATE_PARTITION, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_PARTITIONS_SKEW, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_IN, true)); + defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_OUT, true)); } @Autowired @@ -203,10 +203,9 @@ public class VersionControlManagerImpl implements VersionControlManager { return Result.buildSuc(); } - Set userMetricConfigs = getUserMetricConfig(operator); - for(Map.Entry metricAndShowEntry : metricsSetMap.entrySet()){ - UserMetricConfig userMetricConfig = new UserMetricConfig(type, metricAndShowEntry.getKey(), metricAndShowEntry.getValue(), metricDetailMap.get(metricAndShowEntry.getKey()).getRank()); + for (MetricDetailDTO metricDetailDTO : metricDetailMap.values()) { + UserMetricConfig userMetricConfig = new UserMetricConfig(type, metricDetailDTO.getMetric(), metricDetailDTO.getSet(), metricDetailDTO.getRank()); userMetricConfigs.remove(userMetricConfig); userMetricConfigs.add(userMetricConfig); } @@ -250,15 +249,7 @@ public class VersionControlManagerImpl implements VersionControlManager { return defaultMetrics; } - Set userMetricConfigs = JSON.parseObject(value, new TypeReference>() {}); - - //补充rank不存在情况 - for (UserMetricConfig userMetricConfig : userMetricConfigs) { - if (userMetricConfig.getRank() == 0) { - userMetricConfig.setRank(Constant.DEFAULT_METRIC_RANK); - } - } - return userMetricConfigs; + return JSON.parseObject(value, new TypeReference>() {}); } public static void main(String[] args){ diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java index 9f85d7ca..e244181a 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java @@ -15,20 +15,20 @@ public class UserMetricConfig { private boolean set; - private int rank; + private Integer rank; public UserMetricConfig(int type, String metric, boolean set, Integer rank) { this.type = type; this.metric = metric; this.set = set; - this.rank = rank == null ? Constant.DEFAULT_METRIC_RANK : rank; + this.rank = rank; } public UserMetricConfig(int type, String metric, boolean set) { this.type = type; this.metric = metric; this.set = set; - this.rank = Constant.DEFAULT_METRIC_RANK; + this.rank = null; } @Override diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java index f86c1d8c..edd897ff 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java @@ -42,7 +42,6 @@ public class Constant { */ public static final Integer DEFAULT_CLUSTER_HEALTH_SCORE = 90; - public static final Integer DEFAULT_METRIC_RANK = 10000; public static final String DEFAULT_USER_NAME = "know-streaming-app"; From 725ac10c3d881c00ffefccdfc5b338adbe3db963 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Thu, 22 Sep 2022 11:30:46 +0800 Subject: [PATCH 18/18] =?UTF-8?q?1=E3=80=81=E8=B0=83=E6=95=B4KafkaZKDao?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=EF=BC=9B2=E3=80=81offset=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E6=97=B6=EF=BC=8C=E8=BF=87=E6=BB=A4=E6=8E=89?= =?UTF-8?q?=E6=97=A0leader=E5=88=86=E5=8C=BA=EF=BC=9B3=E3=80=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E9=AA=8C=E8=AF=81ZK=E6=98=AF=E5=90=A6=E5=90=88?= =?UTF-8?q?=E6=B3=95=E6=97=B6=E7=9A=84session=E8=B6=85=E6=97=B6=E6=97=B6?= =?UTF-8?q?=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../km/common/bean/entity/broker/Broker.java | 15 ---------- .../km/common/constant/MsgConstant.java | 4 +++ .../km/core/flusher/zk/AbstractZKWatcher.java | 2 +- .../flusher/zk/handler/AbstractZKHandler.java | 2 +- .../zk/handler/BrokersNodeChangeHandler.java | 2 +- .../ConfigNotificationNodeChangeHandler.java | 8 ++--- .../handler/ControllerNodeChangeHandler.java | 2 +- .../zk/handler/TopicsNodeChangeHandler.java | 2 +- .../broker/impl/BrokerServiceImpl.java | 8 ++--- .../impl/ClusterValidateServiceImpl.java | 4 +-- .../impl/KafkaControllerServiceImpl.java | 2 +- .../partition/impl/PartitionServiceImpl.java | 22 ++++++++++---- .../topic/impl/OpTopicServiceImpl.java | 2 +- .../topic/impl/TopicConfigServiceImpl.java | 2 +- .../service/topic/impl/TopicServiceImpl.java | 2 +- .../kafka/zookeeper/package-info.java | 4 +++ .../zookeeper/service}/KafkaZKDAO.java | 2 +- .../service}/impl/KafkaZKDAOImpl.java | 30 ++++++++++++++----- .../zookeeper/znode/ControllerData.java | 2 +- .../znode/brokers/BrokerMetadata.java | 2 +- .../zookeeper/znode/brokers/PartitionMap.java | 2 +- .../znode/brokers/PartitionState.java | 2 +- .../znode/brokers/TopicMetadata.java | 2 +- .../ConfigChangeNotificationBaseData.java | 2 +- .../ConfigChangeNotificationDataV1.java | 2 +- .../ConfigChangeNotificationDataV2.java | 2 +- .../znode/config/ConfigNodeData.java | 2 +- 27 files changed, 74 insertions(+), 59 deletions(-) create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/package-info.java rename km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/{zk => kafka/zookeeper/service}/KafkaZKDAO.java (97%) rename km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/{zk => kafka/zookeeper/service}/impl/KafkaZKDAOImpl.java (90%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/ControllerData.java (81%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/brokers/BrokerMetadata.java (97%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/brokers/PartitionMap.java (91%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/brokers/PartitionState.java (93%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/brokers/TopicMetadata.java (91%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/config/ConfigChangeNotificationBaseData.java (77%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/config/ConfigChangeNotificationDataV1.java (86%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/config/ConfigChangeNotificationDataV2.java (90%) rename {km-common/src/main/java/com/xiaojukeji/know/streaming/km/common => km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka}/zookeeper/znode/config/ConfigNodeData.java (81%) diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java index fa67cac5..752aade0 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java @@ -5,7 +5,6 @@ import com.alibaba.fastjson.TypeReference; import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData; import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO; import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -79,20 +78,6 @@ public class Broker implements Serializable { return metadata; } - public static Broker buildFrom(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) { - Broker metadata = new Broker(); - metadata.setClusterPhyId(clusterPhyId); - metadata.setBrokerId(brokerId); - metadata.setHost(brokerMetadata.getHost()); - metadata.setPort(brokerMetadata.getPort()); - metadata.setJmxPort(brokerMetadata.getJmxPort()); - metadata.setStartTimestamp(brokerMetadata.getTimestamp()); - metadata.setRack(brokerMetadata.getRack()); - metadata.setStatus(1); - metadata.setEndpointMap(brokerMetadata.getEndpointMap()); - return metadata; - } - public static Broker buildFrom(BrokerPO brokerPO) { Broker broker = ConvertUtil.obj2Obj(brokerPO, Broker.class); String endpointMapStr = brokerPO.getEndpointMap(); diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/MsgConstant.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/MsgConstant.java index 3d0b6a5c..1be8dadf 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/MsgConstant.java +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/MsgConstant.java @@ -52,6 +52,10 @@ public class MsgConstant { /**************************************************** Partition ****************************************************/ + public static String getPartitionNoLeader(Long clusterPhyId, String topicName) { + return String.format("集群ID:[%d] Topic名称:[%s] 所有分区NoLeader", clusterPhyId, topicName); + } + public static String getPartitionNotExist(Long clusterPhyId, String topicName) { return String.format("集群ID:[%d] Topic名称:[%s] 存在非法的分区ID", clusterPhyId, topicName); } diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/AbstractZKWatcher.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/AbstractZKWatcher.java index 261aff0a..e43f1b40 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/AbstractZKWatcher.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/AbstractZKWatcher.java @@ -5,7 +5,7 @@ import com.didiglobal.logi.log.LogFactory; import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; import com.xiaojukeji.know.streaming.km.common.exception.NotExistException; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.KafkaZkClient; import org.springframework.beans.factory.annotation.Autowired; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/AbstractZKHandler.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/AbstractZKHandler.java index e2ed09d1..04a14e87 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/AbstractZKHandler.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/AbstractZKHandler.java @@ -7,7 +7,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; public abstract class AbstractZKHandler { diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/BrokersNodeChangeHandler.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/BrokersNodeChangeHandler.java index b7c93c2f..314195af 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/BrokersNodeChangeHandler.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/BrokersNodeChangeHandler.java @@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.BrokerIdsZNode; import kafka.zookeeper.ZNodeChildChangeHandler; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ConfigNotificationNodeChangeHandler.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ConfigNotificationNodeChangeHandler.java index 91d91571..1e626632 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ConfigNotificationNodeChangeHandler.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ConfigNotificationNodeChangeHandler.java @@ -8,11 +8,11 @@ import com.xiaojukeji.know.streaming.km.common.enums.KafkaConfigTypeEnum; import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum; import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.common.utils.Tuple; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationBaseData; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV1; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV2; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationBaseData; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationDataV1; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationDataV2; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.ConfigEntityChangeNotificationZNode; import kafka.zookeeper.ZNodeChildChangeHandler; import org.apache.zookeeper.data.Stat; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ControllerNodeChangeHandler.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ControllerNodeChangeHandler.java index 904b7d72..b671c4a3 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ControllerNodeChangeHandler.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/ControllerNodeChangeHandler.java @@ -11,7 +11,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils; import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.ControllerZNode; import kafka.zookeeper.ZNodeChangeHandler; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/TopicsNodeChangeHandler.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/TopicsNodeChangeHandler.java index 31602632..88c01281 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/TopicsNodeChangeHandler.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/zk/handler/TopicsNodeChangeHandler.java @@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.TopicsZNode; import kafka.zookeeper.ZNodeChildChangeHandler; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/broker/impl/BrokerServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/broker/impl/BrokerServiceImpl.java index dc702388..fbede23c 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/broker/impl/BrokerServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/broker/impl/BrokerServiceImpl.java @@ -24,7 +24,6 @@ import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistExcept import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap; import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata; import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; @@ -32,8 +31,7 @@ import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient; import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerDAO; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; -import kafka.zk.BrokerIdZNode; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.BrokerIdsZNode; import org.apache.kafka.clients.admin.*; import org.apache.kafka.common.Node; @@ -310,9 +308,7 @@ public class BrokerServiceImpl extends BaseVersionControlService implements Brok List brokerIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), BrokerIdsZNode.path(), false); for (String brokerId: brokerIdList) { - BrokerMetadata metadata = kafkaZKDAO.getData(clusterPhy.getId(), BrokerIdZNode.path(Integer.valueOf(brokerId)), BrokerMetadata.class); - BrokerMetadata.parseAndUpdateBrokerMetadata(metadata); - brokerList.add(Broker.buildFrom(clusterPhy.getId(), Integer.valueOf(brokerId), metadata)); + brokerList.add(kafkaZKDAO.getBrokerMetadata(clusterPhy.getId(), Integer.valueOf(brokerId))); } return Result.buildSuc(brokerList); diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterValidateServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterValidateServiceImpl.java index 6dcd858e..ba72d2fe 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterValidateServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/cluster/impl/ClusterValidateServiceImpl.java @@ -13,8 +13,8 @@ import com.xiaojukeji.know.streaming.km.common.enums.valid.ValidateKafkaAddressE import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterValidateService; import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; -import com.xiaojukeji.know.streaming.km.persistence.zk.impl.KafkaZKDAOImpl; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.impl.KafkaZKDAOImpl; import kafka.server.KafkaConfig; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.*; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/kafkacontroller/impl/KafkaControllerServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/kafkacontroller/impl/KafkaControllerServiceImpl.java index 1fb3f488..8048eabe 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/kafkacontroller/impl/KafkaControllerServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/kafkacontroller/impl/KafkaControllerServiceImpl.java @@ -19,7 +19,7 @@ import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.mysql.kafkacontroller.KafkaControllerDAO; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import org.apache.kafka.clients.admin.*; import org.apache.kafka.common.Node; import org.springframework.beans.factory.annotation.Autowired; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/partition/impl/PartitionServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/partition/impl/PartitionServiceImpl.java index 13eedb41..1795e4d4 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/partition/impl/PartitionServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/partition/impl/PartitionServiceImpl.java @@ -21,14 +21,14 @@ import com.xiaojukeji.know.streaming.km.common.exception.NotExistException; import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException; import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionState; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionMap; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionState; import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaConsumerClient; import com.xiaojukeji.know.streaming.km.persistence.mysql.partition.PartitionDAO; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.TopicPartitionStateZNode; import kafka.zk.TopicPartitionsZNode; import kafka.zk.TopicZNode; @@ -202,10 +202,22 @@ public class PartitionServiceImpl extends BaseVersionControlService implements P @Override public Result> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp) { Map topicPartitionOffsets = new HashMap<>(); - this.listPartitionByTopic(clusterPhyId, topicName) - .stream() + + List partitionList = this.listPartitionByTopic(clusterPhyId, topicName); + if (partitionList == null || partitionList.isEmpty()) { + // Topic不存在 + return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(clusterPhyId, topicName)); + } + + partitionList.stream() + .filter(item -> !item.getLeaderBrokerId().equals(KafkaConstant.NO_LEADER)) .forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec)); + if (topicPartitionOffsets.isEmpty()) { + // 所有分区no-leader + return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FAILED, MsgConstant.getPartitionNoLeader(clusterPhyId, topicName)); + } + try { return (Result>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicName, topicPartitionOffsets, timestamp)); } catch (VCHandlerNotExistException e) { diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/OpTopicServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/OpTopicServiceImpl.java index 7f289c88..7cd017f4 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/OpTopicServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/OpTopicServiceImpl.java @@ -23,7 +23,7 @@ import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.controller.ReplicaAssignment; import kafka.server.ConfigType; import kafka.zk.AdminZkClient; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicConfigServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicConfigServiceImpl.java index 9aaadee5..09be0d43 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicConfigServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicConfigServiceImpl.java @@ -30,7 +30,7 @@ import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.server.ConfigType; import kafka.zk.AdminZkClient; import kafka.zk.KafkaZkClient; diff --git a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicServiceImpl.java b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicServiceImpl.java index bffabec8..e2870d9d 100644 --- a/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicServiceImpl.java +++ b/km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicServiceImpl.java @@ -23,7 +23,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.mysql.topic.TopicDAO; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.zk.TopicsZNode; import org.apache.kafka.clients.admin.*; import org.apache.kafka.common.TopicPartitionInfo; diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/package-info.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/package-info.java new file mode 100644 index 00000000..88139db3 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/package-info.java @@ -0,0 +1,4 @@ +/** + * 读取Kafka在ZK中存储的数据的包 + */ +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper; \ No newline at end of file diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/zk/KafkaZKDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/service/KafkaZKDAO.java similarity index 97% rename from km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/zk/KafkaZKDAO.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/service/KafkaZKDAO.java index 3e00e558..7a7d4b76 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/zk/KafkaZKDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/service/KafkaZKDAO.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.persistence.zk; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service; import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController; diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/zk/impl/KafkaZKDAOImpl.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/service/impl/KafkaZKDAOImpl.java similarity index 90% rename from km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/zk/impl/KafkaZKDAOImpl.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/service/impl/KafkaZKDAOImpl.java index 61a7bad0..82cb8130 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/zk/impl/KafkaZKDAOImpl.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/service/impl/KafkaZKDAOImpl.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.persistence.zk.impl; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.impl; import com.alibaba.fastjson.JSON; import com.didiglobal.logi.log.ILog; @@ -11,11 +11,11 @@ import com.xiaojukeji.know.streaming.km.common.enums.topic.TopicTypeEnum; import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException; import com.xiaojukeji.know.streaming.km.common.exception.NotExistException; import com.xiaojukeji.know.streaming.km.common.utils.Tuple; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.ControllerData; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata; -import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.ControllerData; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.BrokerMetadata; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionMap; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; -import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; +import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO; import kafka.utils.Json; import kafka.zk.*; import kafka.zookeeper.AsyncResponse; @@ -46,14 +46,14 @@ public class KafkaZKDAOImpl implements KafkaZKDAO { public Broker getBrokerMetadata(String zkAddress) throws KeeperException.NoNodeException, AdminOperateException { ZooKeeper zooKeeper = null; try { - zooKeeper = new ZooKeeper(zkAddress, 1000, watchedEvent -> logger.info(" receive event : " + watchedEvent.getType().name())); + zooKeeper = new ZooKeeper(zkAddress, 3000, watchedEvent -> logger.info(" receive event : " + watchedEvent.getType().name())); List brokerIdList = this.getChildren(zooKeeper, BrokerIdsZNode.path()); if (brokerIdList == null || brokerIdList.isEmpty()) { return null; } BrokerMetadata brokerMetadata = this.getData(zooKeeper, BrokerIdZNode.path(Integer.parseInt(brokerIdList.get(0))), false, BrokerMetadata.class); - return Broker.buildFrom(null, Integer.valueOf(brokerIdList.get(0)), brokerMetadata); + return this.convert2Broker(null, Integer.valueOf(brokerIdList.get(0)), brokerMetadata); } catch (KeeperException.NoNodeException nne) { logger.warn("method=getBrokerMetadata||zkAddress={}||errMsg=exception", zkAddress, nne); throw nne; @@ -79,7 +79,7 @@ public class KafkaZKDAOImpl implements KafkaZKDAO { try { BrokerMetadata metadata = this.getData(kafkaZkClient.currentZooKeeper(), BrokerIdZNode.path(brokerId), false, BrokerMetadata.class); BrokerMetadata.parseAndUpdateBrokerMetadata(metadata); - return Broker.buildFrom(clusterPhyId, brokerId, metadata); + return this.convert2Broker(clusterPhyId, brokerId, metadata); } catch (KeeperException ke) { logger.error("method=getBrokerMetadata||clusterPhyId={}||brokerId={}||errMsg=exception", clusterPhyId, brokerId, ke); throw ke; @@ -269,4 +269,18 @@ public class KafkaZKDAOImpl implements KafkaZKDAO { byte[] bytes = zooKeeper.getData(path, addWatch, null); return JSON.parseObject(bytes, clazz); } + + private Broker convert2Broker(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) { + Broker metadata = new Broker(); + metadata.setClusterPhyId(clusterPhyId); + metadata.setBrokerId(brokerId); + metadata.setHost(brokerMetadata.getHost()); + metadata.setPort(brokerMetadata.getPort()); + metadata.setJmxPort(brokerMetadata.getJmxPort()); + metadata.setStartTimestamp(brokerMetadata.getTimestamp()); + metadata.setRack(brokerMetadata.getRack()); + metadata.setStatus(1); + metadata.setEndpointMap(brokerMetadata.getEndpointMap()); + return metadata; + } } diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/ControllerData.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/ControllerData.java similarity index 81% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/ControllerData.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/ControllerData.java index f69c6862..afc7f55b 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/ControllerData.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/ControllerData.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/BrokerMetadata.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/BrokerMetadata.java similarity index 97% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/BrokerMetadata.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/BrokerMetadata.java index 480867af..3b252c5f 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/BrokerMetadata.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/BrokerMetadata.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionMap.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/PartitionMap.java similarity index 91% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionMap.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/PartitionMap.java index bf1fbd1a..4bc36cac 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionMap.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/PartitionMap.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionState.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/PartitionState.java similarity index 93% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionState.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/PartitionState.java index 60ae4307..47be5cb9 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionState.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/PartitionState.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/TopicMetadata.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/TopicMetadata.java similarity index 91% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/TopicMetadata.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/TopicMetadata.java index 803a5e29..f84c8fcf 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/TopicMetadata.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/brokers/TopicMetadata.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationBaseData.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationBaseData.java similarity index 77% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationBaseData.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationBaseData.java index 86a3abe9..09ffee10 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationBaseData.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationBaseData.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV1.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationDataV1.java similarity index 86% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV1.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationDataV1.java index 75598e65..1853b940 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV1.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationDataV1.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV2.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationDataV2.java similarity index 90% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV2.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationDataV2.java index 6b0d8806..5e6024fa 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV2.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigChangeNotificationDataV2.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigNodeData.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigNodeData.java similarity index 81% rename from km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigNodeData.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigNodeData.java index 13132b4f..287912dc 100644 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigNodeData.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/kafka/zookeeper/znode/config/ConfigNodeData.java @@ -1,4 +1,4 @@ -package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; +package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config; import lombok.AllArgsConstructor; import lombok.Data;