mirror of
https://github.com/didi/KnowStreaming.git
synced 2025-12-24 20:22:12 +08:00
1. 调整docker-compose.yml中各个服务的镜像
2. 经过@wangdongfang-aden大哥的调试将helm与docker镜像合二为一，于是删减掉各个镜像的Dockerfile与启动脚本，后续也不需要额外维护
This commit is contained in:
haoqi
@@ -3,7 +3,7 @@ version: "3"
|
||||
services:
|
||||
|
||||
knowstreaming-manager:
|
||||
image: haiti/knowstreaming-manager:0.1.0
|
||||
image: knowstreaming/knowstreaming-manager:0.2.0-test
|
||||
container_name: knowstreaming-manager
|
||||
privileged: true
|
||||
restart: always
|
||||
@@ -12,6 +12,9 @@ services:
|
||||
- knowstreaming-mysql
|
||||
expose:
|
||||
- 80
|
||||
command:
|
||||
- /bin/sh
|
||||
- /ks-start.sh
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
|
||||
@@ -27,9 +30,9 @@ services:
|
||||
# extra_hosts:
|
||||
# - "hostname:x.x.x.x"
|
||||
# volumes:
|
||||
# - /Users/haoqi/Downloads/ks/manage/log:/logs
|
||||
# - /ks/manage/log:/logs
|
||||
knowstreaming-ui:
|
||||
image: haiti/knowstreaming-ui:0.1.0
|
||||
image: knowstreaming/knowstreaming-ui:0.2.0-test1
|
||||
container_name: knowstreaming-ui
|
||||
restart: always
|
||||
ports:
|
||||
@@ -56,27 +59,33 @@ services:
|
||||
ES_JAVA_OPTS: -Xms512m -Xmx512m
|
||||
discovery.type: single-node
|
||||
# volumes:
|
||||
# - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data
|
||||
# - /ks/es/data:/usr/share/elasticsearch/data
|
||||
|
||||
knowstreaming-init:
|
||||
image: haiti/knowstreaming-init:0.1.0
|
||||
image: knowstreaming/knowstreaming-manager:0.2.0-test
|
||||
container_name: knowstreaming_init
|
||||
depends_on:
|
||||
- elasticsearch-single
|
||||
command:
|
||||
- /bin/bash
|
||||
- /es_template_create.sh
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
SERVER_ES_ADDRESS: elasticsearch-single:9200
|
||||
|
||||
|
||||
knowstreaming-mysql:
|
||||
image: haiti/knowstreaming-mysql:0.1.0
|
||||
image: knowstreaming/knowstreaming-mysql:0.2.0-test
|
||||
container_name: knowstreaming-mysql
|
||||
restart: always
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
MYSQL_ROOT_PASSWORD: admin2022_
|
||||
MYSQL_DATABASE: know_streaming
|
||||
MYSQL_ROOT_HOST: '%'
|
||||
expose:
|
||||
- 3306
|
||||
# ports:
|
||||
# - '3306:3306'
|
||||
# volumes:
|
||||
# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql
|
||||
# - /ks/mysql/data:/data/mysql
|
||||
@@ -1,10 +0,0 @@
|
||||
FROM debian:stable-20220912-slim
|
||||
LABEL org.opencontainers.image.authors="haoqi123"
|
||||
ARG INIT_ES_FILE=./init/*
|
||||
COPY ${INIT_ES_FILE} /
|
||||
ENV TZ=Asia/Shanghai
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y curl
|
||||
|
||||
ENTRYPOINT ["/bin/bash","/template.sh","start"]
|
||||
@@ -1,661 +0,0 @@
|
||||
echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}"
|
||||
while true
|
||||
do
|
||||
curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1
|
||||
if [ "$?" != "0" ];then
|
||||
sleep 1s
|
||||
else
|
||||
echo "ElasticSearch Start Initialize"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_broker_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"brokerId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"routingValue" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"clusterPhyId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"metrics" : {
|
||||
"properties" : {
|
||||
"NetworkProcessorAvgIdle" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"UnderReplicatedPartitions" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesIn_min_15" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthCheckTotal" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"RequestHandlerAvgIdle" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"connectionsCount" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesIn_min_5" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthScore" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesOut" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesOut_min_15" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesIn" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesOut_min_5" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"TotalRequestQueueSize" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"MessagesIn" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"TotalProduceRequests" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthCheckPassed" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"TotalResponseQueueSize" : {
|
||||
"type" : "float"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"index" : true,
|
||||
"type" : "date",
|
||||
"doc_values" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}'
|
||||
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_cluster_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"routingValue" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"clusterPhyId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"metrics" : {
|
||||
"properties" : {
|
||||
"Connections" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"BytesIn_min_15" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"PartitionURP" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthScore_Topics" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"EventQueueSize" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"ActiveControllerCount" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"GroupDeads" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"BytesIn_min_5" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckTotal_Topics" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Partitions" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"BytesOut" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Groups" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"BytesOut_min_15" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"TotalRequestQueueSize" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckPassed_Groups" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"TotalProduceRequests" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckPassed" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"TotalLogSize" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"GroupEmptys" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"PartitionNoLeader" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthScore_Brokers" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Messages" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Topics" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"PartitionMinISR_E" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckTotal" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Brokers" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Replicas" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckTotal_Groups" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"GroupRebalances" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"MessageIn" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthScore" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckPassed_Topics" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckTotal_Brokers" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"PartitionMinISR_S" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"BytesIn" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"BytesOut_min_5" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"GroupActives" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"MessagesIn" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"GroupReBalances" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckPassed_Brokers" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthScore_Groups" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"TotalResponseQueueSize" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"Zookeepers" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"LeaderMessages" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthScore_Cluster" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckPassed_Cluster" : {
|
||||
"type" : "double"
|
||||
},
|
||||
"HealthCheckTotal_Cluster" : {
|
||||
"type" : "double"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"type" : "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}'
|
||||
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_group_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"group" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"partitionId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"routingValue" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"clusterPhyId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"topic" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"metrics" : {
|
||||
"properties" : {
|
||||
"HealthScore" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"Lag" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"OffsetConsumed" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthCheckTotal" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthCheckPassed" : {
|
||||
"type" : "float"
|
||||
}
|
||||
}
|
||||
},
|
||||
"groupMetric" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"key" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"index" : true,
|
||||
"type" : "date",
|
||||
"doc_values" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}'
|
||||
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_partition_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"brokerId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"partitionId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"routingValue" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"clusterPhyId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"topic" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"metrics" : {
|
||||
"properties" : {
|
||||
"LogStartOffset" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"Messages" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"LogEndOffset" : {
|
||||
"type" : "float"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"index" : true,
|
||||
"type" : "date",
|
||||
"doc_values" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}'
|
||||
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_partition_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"brokerId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"partitionId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"routingValue" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"clusterPhyId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"topic" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"metrics" : {
|
||||
"properties" : {
|
||||
"LogStartOffset" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"Messages" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"LogEndOffset" : {
|
||||
"type" : "float"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"index" : true,
|
||||
"type" : "date",
|
||||
"doc_values" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}[root@10-255-0-23 template]# cat ks_kafka_replication_metric
|
||||
PUT _template/ks_kafka_replication_metric
|
||||
{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_replication_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"index" : true,
|
||||
"type" : "date",
|
||||
"doc_values" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}'
|
||||
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{
|
||||
"order" : 10,
|
||||
"index_patterns" : [
|
||||
"ks_kafka_topic_metric*"
|
||||
],
|
||||
"settings" : {
|
||||
"index" : {
|
||||
"number_of_shards" : "10"
|
||||
}
|
||||
},
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"brokerId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"routingValue" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"topic" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"clusterPhyId" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"metrics" : {
|
||||
"properties" : {
|
||||
"BytesIn_min_15" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"Messages" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesRejected" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"PartitionURP" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthCheckTotal" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"ReplicationCount" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"ReplicationBytesOut" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"ReplicationBytesIn" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"FailedFetchRequests" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesIn_min_5" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthScore" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"LogSize" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesOut" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesOut_min_15" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"FailedProduceRequests" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesIn" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"BytesOut_min_5" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"MessagesIn" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"TotalProduceRequests" : {
|
||||
"type" : "float"
|
||||
},
|
||||
"HealthCheckPassed" : {
|
||||
"type" : "float"
|
||||
}
|
||||
}
|
||||
},
|
||||
"brokerAgg" : {
|
||||
"type" : "keyword"
|
||||
},
|
||||
"key" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"ignore_above" : 256,
|
||||
"type" : "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timestamp" : {
|
||||
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
|
||||
"index" : true,
|
||||
"type" : "date",
|
||||
"doc_values" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aliases" : { }
|
||||
}'
|
||||
|
||||
for i in {0..6};
|
||||
do
|
||||
logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
|
||||
curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \
|
||||
exit 2
|
||||
done
|
||||
|
||||
echo "ElasticSearch Initialize Success"
|
||||
@@ -1,38 +0,0 @@
|
||||
FROM docker.io/amd64/eclipse-temurin:17.0.4.1_1-jdk-centos7
|
||||
|
||||
LABEL maintainer="haoqi123 <1148648445@qq.com>"
|
||||
|
||||
WORKDIR /knowstreaming
|
||||
|
||||
ARG VERSION_MAVEN=3.6.3
|
||||
ARG VERSION_KM=3.0.0-beta.2
|
||||
ARG INIT_FILE=./init/*
|
||||
ARG CONFIG_FILE=./conf/*
|
||||
|
||||
COPY ${INIT_FILE} /
|
||||
COPY ${CONFIG_FILE} /conf/
|
||||
|
||||
# SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
RUN yum install -y git
|
||||
|
||||
RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_MAVEN}/binaries/apache-maven-${VERSION_MAVEN}-bin.tar.gz" \
|
||||
&& mkdir maven/ \
|
||||
&& tar -zxf "apache-maven-${VERSION_MAVEN}-bin.tar.gz" -C maven/ --strip-components=1 --no-same-owner \
|
||||
&& rm -rf /knowstreaming/*.tar.gz \
|
||||
&& ln -s /knowstreaming/maven/bin/mvn /usr/local/bin/mvn \
|
||||
# && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \
|
||||
&& git clone https://github.com/didi/KnowStreaming.git \
|
||||
# && unzip -o -q master.zip \
|
||||
&& cd KnowStreaming && git checkout v${VERSION_KM} \
|
||||
&& sed -i "/km-console/d" ./pom.xml \
|
||||
&& mvn -U clean package -Dmaven.test.skip=true \
|
||||
&& cd / && mkdir /app \
|
||||
&& mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \
|
||||
&& rm -rf /knowstreaming/* /usr/local/bin/mvn \
|
||||
&& yum erase -y git && yum clean all \
|
||||
&& rm -rf /root/.m2 /tmp/*
|
||||
|
||||
ENTRYPOINT ["/bin/bash","/start.sh"]
|
||||
|
||||
HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1
|
||||
@@ -1,107 +0,0 @@
|
||||
server:
|
||||
port: 80 # 服务端口
|
||||
tomcat:
|
||||
accept-count: 1000
|
||||
max-connections: 10000
|
||||
|
||||
spring:
|
||||
application:
|
||||
name: know-streaming
|
||||
profiles:
|
||||
active: dev
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
jackson:
|
||||
time-zone: GMT+8
|
||||
datasource:
|
||||
know-streaming: # know-streaming 自身数据库的配置
|
||||
jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: SERVER_MYSQL_USER
|
||||
password: SERVER_MYSQL_PASSWORD
|
||||
driver-class-name: org.mariadb.jdbc.Driver
|
||||
maximum-pool-size: 20
|
||||
idle-timeout: 30000
|
||||
connection-test-query: SELECT 1
|
||||
logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
|
||||
jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: SERVER_MYSQL_USER
|
||||
password: SERVER_MYSQL_PASSWORD
|
||||
driver-class-name: org.mariadb.jdbc.Driver
|
||||
max-lifetime: 60000
|
||||
init-sql: true
|
||||
init-thread-num: 20
|
||||
max-thread-num: 50
|
||||
log-expire: 3 # 日志保存天数，以天为单位
|
||||
app-name: know-streaming
|
||||
claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual
|
||||
logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
|
||||
jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: SERVER_MYSQL_USER
|
||||
password: SERVER_MYSQL_PASSWORD
|
||||
driver-class-name: org.mariadb.jdbc.Driver
|
||||
app-name: know-streaming
|
||||
resource-extend-bean-name: myResourceExtendImpl
|
||||
login-extend-bean-name: logiSecurityDefaultLoginExtendImpl
|
||||
|
||||
logging:
|
||||
config: classpath:logback-spring.xml
|
||||
|
||||
# 线程池大小相关配置
|
||||
thread-pool:
|
||||
scheduled:
|
||||
thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
|
||||
|
||||
collector: # 采集模块的配置
|
||||
future-util: # 采集模块线程池配置
|
||||
num: 3 # 线程池个数
|
||||
thread-num: 64 # 每个线程池核心线程数
|
||||
queue-size: 10000 # 每个线程池队列大小
|
||||
select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改
|
||||
suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改
|
||||
|
||||
task: # 任务模块的配置
|
||||
metrics: # metrics采集任务配置
|
||||
thread-num: 18 # metrics采集任务线程池核心线程数
|
||||
queue-size: 180 # metrics采集任务线程池队列大小
|
||||
metadata: # metadata同步任务配置
|
||||
thread-num: 27 # metadata同步任务线程池核心线程数
|
||||
queue-size: 270 # metadata同步任务线程池队列大小
|
||||
common: # 剩余其他任务配置
|
||||
thread-num: 15 # 剩余其他任务线程池核心线程数
|
||||
queue-size: 150 # 剩余其他任务线程池队列大小
|
||||
|
||||
|
||||
# 客户端池大小相关配置
|
||||
client-pool:
|
||||
kafka-consumer:
|
||||
min-idle-client-num: 2 # 最小空闲客户端数
|
||||
max-idle-client-num: 20 # 最大空闲客户端数
|
||||
max-total-client-num: 20 # 最大客户端数
|
||||
borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒
|
||||
|
||||
|
||||
# ES客户端配置
|
||||
es:
|
||||
client:
|
||||
address: SERVER_ES_ADDRESS
|
||||
client-cnt: 10
|
||||
io-thread-cnt: 2
|
||||
max-retry-cnt: 5
|
||||
|
||||
# 普罗米修斯指标导出相关配置
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
base-path: /metrics
|
||||
exposure:
|
||||
include: '*'
|
||||
health:
|
||||
elasticsearch:
|
||||
enabled: false
|
||||
metrics:
|
||||
export:
|
||||
prometheus:
|
||||
descriptions: true
|
||||
enabled: true
|
||||
tags:
|
||||
application: know-streaming
|
||||
@@ -1,7 +0,0 @@
|
||||
sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml
|
||||
sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml
|
||||
sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml
|
||||
sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml
|
||||
sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml
|
||||
|
||||
java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /app/app.jar --spring.config.location=/conf/application.yml
|
||||
@@ -1,10 +0,0 @@
|
||||
FROM mysql:5.7.39-oracle
|
||||
|
||||
LABEL org.opencontainers.image.authors="haoqi123"
|
||||
|
||||
EXPOSE 3306
|
||||
|
||||
ARG INIT_SOURCE_FILE=./sql/*.sql
|
||||
ARG MYSQL_CNF_FILE=./conf/*.cnf
|
||||
COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/
|
||||
COPY ${MYSQL_CNF_FILE} /etc/mysql/conf.d/
|
||||
@@ -1,15 +0,0 @@
|
||||
[client]
|
||||
default-character-set=utf8
|
||||
[mysql]
|
||||
default-character-set=utf8
|
||||
[mysqld]
|
||||
skip-host-cache
|
||||
skip-name-resolve
|
||||
datadir=/data/mysql
|
||||
socket=/var/lib/mysql/mysql.sock
|
||||
secure-file-priv=/var/lib/mysql-files
|
||||
character-set-server=utf8
|
||||
user=mysql
|
||||
symbolic-links=0
|
||||
pid-file=/var/run/mysqld/mysqld.pid
|
||||
sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
|
||||
@@ -1,86 +0,0 @@
|
||||
FROM docker.io/amd64/nginx:1.22
|
||||
|
||||
LABEL maintainer="haoqi123 <1148648445@qq.com>"
|
||||
|
||||
WORKDIR /knowstreaming
|
||||
|
||||
ARG VERSION_NODE=14.20.0
|
||||
ARG VERSION_KM=3.0.0-beta.2
|
||||
|
||||
# SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
RUN apt update \
|
||||
&& apt install -y git unzip
|
||||
|
||||
RUN echo 'server { \
|
||||
listen 80; \
|
||||
server_name localhost; \
|
||||
gzip on; \
|
||||
gzip_buffers 16 8k; \
|
||||
gzip_comp_level 4; \
|
||||
gzip_http_version 1.0; \
|
||||
gzip_min_length 1280; \
|
||||
gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; \
|
||||
gzip_vary on; \
|
||||
root /pub; \
|
||||
location / { \
|
||||
root /pub; \
|
||||
if ($request_filename ~* .*\.(?:htm|html|json)$) { \
|
||||
add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; \
|
||||
} \
|
||||
try_files $uri /layout/index.html; \
|
||||
} \
|
||||
location ~* \.(json)$ { \
|
||||
add_header Cache-Control no-cache; \
|
||||
} \
|
||||
location @kmfallback { \
|
||||
} \
|
||||
#location ~ ^/(clusters|config|cluster|login) { \
|
||||
# rewrite ^.*$ /; \
|
||||
#} \
|
||||
location ~ ^/ks-km/api/v3 { \
|
||||
#rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; \
|
||||
proxy_pass http://knowstreaming-manager; \
|
||||
#proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \
|
||||
#proxy_cookie_path /ks-km/ /; \
|
||||
#proxy_set_header Host $host; \
|
||||
#proxy_set_header Referer $http_referer; \
|
||||
#proxy_set_header Cookie $http_cookie; \
|
||||
#proxy_set_header X-Real-Ip $remote_addr; \
|
||||
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; \
|
||||
} \
|
||||
location ~ ^/logi-security/api/v1 { \
|
||||
#rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; \
|
||||
proxy_pass http://knowstreaming-manager; \
|
||||
#proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \
|
||||
} \
|
||||
location ~ ^/(401|403|404|500){ \
|
||||
rewrite ^.*$ /; \
|
||||
} \
|
||||
}' > /etc/nginx/conf.d/default.conf
|
||||
|
||||
RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/v${VERSION_NODE}/node-v${VERSION_NODE}-linux-x64.tar.gz" \
|
||||
&& mkdir node/ \
|
||||
&& tar -zxf "node-v${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \
|
||||
&& rm -rf /knowstreaming/*.tar.gz \
|
||||
&& ln -s /knowstreaming/node/bin/node /usr/local/bin/node \
|
||||
&& ln -s /knowstreaming/node/bin/npm /usr/local/bin/npm \
|
||||
&& npm config set strict-ssl false \
|
||||
&& npm config set registry https://registry.npm.taobao.org \
|
||||
&& npm info underscore \
|
||||
&& node --version \
|
||||
&& npm --version \
|
||||
&& curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/v${VERSION_KM}.zip \
|
||||
&& unzip -o -q v${VERSION_KM}.zip \
|
||||
&& cd KnowStreaming-${VERSION_KM}/km-console \
|
||||
&& npm install -g lerna && npm run i && npm run build \
|
||||
&& mv /knowstreaming/KnowStreaming-${VERSION_KM}/km-rest/src/main/resources/templates /pub \
|
||||
&& npm uninstall -g lerna \
|
||||
&& cd / \
|
||||
&& rm -rf /knowstreaming/* /usr/local/bin/node /usr/local/bin/npm \
|
||||
&& apt remove -y git unzip && apt autoremove -y && apt remove --purge --auto-remove -y \
|
||||
&& rm -rf /var/lib/apt/lists/* /tmp /root/.npm
|
||||
|
||||
#ENTRYPOINT ["ls", "/ks_ui/km-console"]
|
||||
|
||||
#CMD ["nginx", "-g", "daemon off;"]
|
||||
Reference in New Issue
Block a user