diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index d916a51e..a9fe2bf3 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: knowstreaming-manager: - image: haiti/knowstreaming-manager:0.1.0 + image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming-manager privileged: true restart: always @@ -12,6 +12,9 @@ services: - knowstreaming-mysql expose: - 80 + command: + - /bin/sh + - /ks-start.sh environment: TZ: Asia/Shanghai @@ -27,9 +30,9 @@ services: # extra_hosts: # - "hostname:x.x.x.x" # volumes: -# - /Users/haoqi/Downloads/ks/manage/log:/logs +# - /ks/manage/log:/logs knowstreaming-ui: - image: haiti/knowstreaming-ui:0.1.0 + image: knowstreaming/knowstreaming-ui:0.2.0-test1 container_name: knowstreaming-ui restart: always ports: @@ -56,27 +59,33 @@ services: ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node # volumes: -# - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data +# - /ks/es/data:/usr/share/elasticsearch/data knowstreaming-init: - image: haiti/knowstreaming-init:0.1.0 + image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming_init depends_on: - elasticsearch-single + command: + - /bin/bash + - /es_template_create.sh environment: TZ: Asia/Shanghai SERVER_ES_ADDRESS: elasticsearch-single:9200 knowstreaming-mysql: - image: haiti/knowstreaming-mysql:0.1.0 + image: knowstreaming/knowstreaming-mysql:0.2.0-test container_name: knowstreaming-mysql restart: always environment: TZ: Asia/Shanghai MYSQL_ROOT_PASSWORD: admin2022_ MYSQL_DATABASE: know_streaming + MYSQL_ROOT_HOST: '%' expose: - 3306 +# ports: +# - '3306:3306' # volumes: -# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql \ No newline at end of file +# - /ks/mysql/data:/data/mysql \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/Dockerfile b/km-dist/docker/knowstreaming-init/Dockerfile deleted file mode 100644 index ddb2120c..00000000 --- a/km-dist/docker/knowstreaming-init/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM debian:stable-20220912-slim -LABEL org.opencontainers.image.authors="haoqi123" -ARG INIT_ES_FILE=./init/* -COPY ${INIT_ES_FILE} / -ENV TZ=Asia/Shanghai - -RUN apt-get update \ - && apt-get install -y curl - -ENTRYPOINT ["/bin/bash","/template.sh","start"] \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-init/init/template.sh b/km-dist/docker/knowstreaming-init/init/template.sh deleted file mode 100644 index ce41db73..00000000 --- a/km-dist/docker/knowstreaming-init/init/template.sh +++ /dev/null @@ -1,661 +0,0 @@ -echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}" -while true -do - curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 - if [ "$?" != "0" ];then - sleep 1s - else - echo "ElasticSearch Start Initialize" - break - fi -done - -curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_broker_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "NetworkProcessorAvgIdle" : { - "type" : "float" - }, - "UnderReplicatedPartitions" : { - "type" : "float" - }, - "BytesIn_min_15" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "RequestHandlerAvgIdle" : { - "type" : "float" - }, - "connectionsCount" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "TotalRequestQueueSize" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "TotalResponseQueueSize" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "Connections" : { - "type" : "double" - }, - "BytesIn_min_15" : { - "type" : "double" - }, - "PartitionURP" : { - "type" : "double" - }, - "HealthScore_Topics" : { - "type" : "double" - }, - "EventQueueSize" : { - "type" : "double" - }, - "ActiveControllerCount" : { - "type" : "double" - }, - "GroupDeads" : { - "type" : "double" - }, - "BytesIn_min_5" : { - "type" : "double" - }, - "HealthCheckTotal_Topics" : { - "type" : "double" - }, - "Partitions" : { - "type" : "double" - }, - "BytesOut" : { - "type" : "double" - }, - "Groups" : { - "type" : "double" - }, - "BytesOut_min_15" : { - "type" : "double" - }, - "TotalRequestQueueSize" : { - "type" : "double" - }, - "HealthCheckPassed_Groups" : { - "type" : "double" - }, - "TotalProduceRequests" : { - "type" : "double" - }, - "HealthCheckPassed" : { - "type" : "double" - }, - "TotalLogSize" : { - "type" : "double" - }, - "GroupEmptys" : { - "type" : "double" - }, - "PartitionNoLeader" : { - "type" : "double" - }, - "HealthScore_Brokers" : { - "type" : "double" - }, - "Messages" : { - "type" : "double" - }, - "Topics" : { - "type" : "double" - }, - "PartitionMinISR_E" : { - "type" : "double" - }, - "HealthCheckTotal" : { - "type" : "double" - }, - "Brokers" : { - "type" : "double" - }, - "Replicas" : { - "type" : "double" - }, - "HealthCheckTotal_Groups" : { - "type" : "double" - }, - "GroupRebalances" : { - "type" : "double" - }, - "MessageIn" : { - "type" : "double" - }, - "HealthScore" : { - "type" : "double" - }, - "HealthCheckPassed_Topics" : { - "type" : "double" - }, - "HealthCheckTotal_Brokers" : { - "type" : "double" - }, - "PartitionMinISR_S" : { - "type" : "double" - }, - "BytesIn" : { - "type" : "double" - }, - "BytesOut_min_5" : { - "type" : "double" - }, - "GroupActives" : { - "type" : "double" - }, - "MessagesIn" : { - "type" : "double" - }, - "GroupReBalances" : { - "type" : "double" - }, - "HealthCheckPassed_Brokers" : { - "type" : "double" - }, - "HealthScore_Groups" : { - "type" : "double" - }, - "TotalResponseQueueSize" : { - "type" : "double" - }, - "Zookeepers" : { - "type" : "double" - }, - "LeaderMessages" : { - "type" : "double" - }, - "HealthScore_Cluster" : { - "type" : "double" - }, - "HealthCheckPassed_Cluster" : { - "type" : "double" - }, - "HealthCheckTotal_Cluster" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_group_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "group" : { - "type" : "keyword" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "HealthScore" : { - "type" : "float" - }, - "Lag" : { - "type" : "float" - }, - "OffsetConsumed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "groupMetric" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }[root@10-255-0-23 template]# cat ks_kafka_replication_metric -PUT _template/ks_kafka_replication_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_replication_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_topic_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "topic" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "BytesIn_min_15" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "BytesRejected" : { - "type" : "float" - }, - "PartitionURP" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ReplicationCount" : { - "type" : "float" - }, - "ReplicationBytesOut" : { - "type" : "float" - }, - "ReplicationBytesIn" : { - "type" : "float" - }, - "FailedFetchRequests" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "LogSize" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "FailedProduceRequests" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "brokerAgg" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -for i in {0..6}; -do - logdate=_$(date -d "${i} day ago" +%Y-%m-%d) - curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ - exit 2 -done - -echo "ElasticSearch Initialize Success" \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile deleted file mode 100644 index b522eb5c..00000000 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM docker.io/amd64/eclipse-temurin:17.0.4.1_1-jdk-centos7 - -LABEL maintainer="haoqi123 <1148648445@qq.com>" - -WORKDIR /knowstreaming - -ARG VERSION_MAVEN=3.6.3 -ARG VERSION_KM=3.0.0-beta.2 -ARG INIT_FILE=./init/* -ARG CONFIG_FILE=./conf/* - -COPY ${INIT_FILE} / -COPY ${CONFIG_FILE} /conf/ - -# SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN yum install -y git - -RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_MAVEN}/binaries/apache-maven-${VERSION_MAVEN}-bin.tar.gz" \ - && mkdir maven/ \ - && tar -zxf "apache-maven-${VERSION_MAVEN}-bin.tar.gz" -C maven/ --strip-components=1 --no-same-owner \ - && rm -rf /knowstreaming/*.tar.gz \ - && ln -s /knowstreaming/maven/bin/mvn /usr/local/bin/mvn \ -# && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \ - && git clone https://github.com/didi/KnowStreaming.git \ -# && unzip -o -q master.zip \ - && cd KnowStreaming && git checkout v${VERSION_KM} \ - && sed -i "/km-console/d" ./pom.xml \ - && mvn -U clean package -Dmaven.test.skip=true \ - && cd / && mkdir /app \ - && mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \ - && rm -rf /knowstreaming/* /usr/local/bin/mvn \ - && yum erase -y git && yum clean all \ - && rm -rf /root/.m2 /tmp/* - -ENTRYPOINT ["/bin/bash","/start.sh"] - -HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1 diff --git a/km-dist/docker/knowstreaming-manage/conf/application.yml b/km-dist/docker/knowstreaming-manage/conf/application.yml deleted file mode 100644 index 80323cd0..00000000 --- a/km-dist/docker/knowstreaming-manage/conf/application.yml +++ /dev/null @@ -1,107 +0,0 @@ -server: - port: 80 # 服务端口 - tomcat: - accept-count: 1000 - max-connections: 10000 - -spring: - application: - name: know-streaming - profiles: - active: dev - main: - allow-bean-definition-overriding: true - jackson: - time-zone: GMT+8 - datasource: - know-streaming: # know-streaming 自身数据库的配置 - jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - username: SERVER_MYSQL_USER - password: SERVER_MYSQL_PASSWORD - driver-class-name: org.mariadb.jdbc.Driver - maximum-pool-size: 20 - idle-timeout: 30000 - connection-test-query: SELECT 1 - logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 - jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - username: SERVER_MYSQL_USER - password: SERVER_MYSQL_PASSWORD - driver-class-name: org.mariadb.jdbc.Driver - max-lifetime: 60000 - init-sql: true - init-thread-num: 20 - max-thread-num: 50 - log-expire: 3 # 日志保存天数，以天为单位 - app-name: know-streaming - claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual - logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 - jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - username: SERVER_MYSQL_USER - password: SERVER_MYSQL_PASSWORD - driver-class-name: org.mariadb.jdbc.Driver - app-name: know-streaming - resource-extend-bean-name: myResourceExtendImpl - login-extend-bean-name: logiSecurityDefaultLoginExtendImpl - -logging: - config: classpath:logback-spring.xml - -# 线程池大小相关配置 -thread-pool: - scheduled: - thread-num: 2 # @Scheduled任务的线程池大小，默认是一个 - - collector: # 采集模块的配置 - future-util: # 采集模块线程池配置 - num: 3 # 线程池个数 - thread-num: 64 # 每个线程池核心线程数 - queue-size: 10000 # 每个线程池队列大小 - select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改 - suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改 - - task: # 任务模块的配置 - metrics: # metrics采集任务配置 - thread-num: 18 # metrics采集任务线程池核心线程数 - queue-size: 180 # metrics采集任务线程池队列大小 - metadata: # metadata同步任务配置 - thread-num: 27 # metadata同步任务线程池核心线程数 - queue-size: 270 # metadata同步任务线程池队列大小 - common: # 剩余其他任务配置 - thread-num: 15 # 剩余其他任务线程池核心线程数 - queue-size: 150 # 剩余其他任务线程池队列大小 - - -# 客户端池大小相关配置 -client-pool: - kafka-consumer: - min-idle-client-num: 2 # 最小空闲客户端数 - max-idle-client-num: 20 # 最大空闲客户端数 - max-total-client-num: 20 # 最大客户端数 - borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒 - - -# ES客户端配置 -es: - client: - address: SERVER_ES_ADDRESS - client-cnt: 10 - io-thread-cnt: 2 - max-retry-cnt: 5 - -# 普罗米修斯指标导出相关配置 -management: - endpoints: - web: - base-path: /metrics - exposure: - include: '*' - health: - elasticsearch: - enabled: false - metrics: - export: - prometheus: - descriptions: true - enabled: true - tags: - application: know-streaming \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/init/start.sh b/km-dist/docker/knowstreaming-manage/init/start.sh deleted file mode 100644 index 79c5d77a..00000000 --- a/km-dist/docker/knowstreaming-manage/init/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml -sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml - -java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile deleted file mode 100644 index 95720f05..00000000 --- a/km-dist/docker/knowstreaming-mysql/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM mysql:5.7.39-oracle - -LABEL org.opencontainers.image.authors="haoqi123" - -EXPOSE 3306 - -ARG INIT_SOURCE_FILE=./sql/*.sql -ARG MYSQL_CNF_FILE=./conf/*.cnf -COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/ -COPY ${MYSQL_CNF_FILE} /etc/mysql/conf.d/ \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf b/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf deleted file mode 100644 index 37c97010..00000000 --- a/km-dist/docker/knowstreaming-mysql/conf/mysql.cnf +++ /dev/null @@ -1,15 +0,0 @@ -[client] -default-character-set=utf8 -[mysql] -default-character-set=utf8 -[mysqld] -skip-host-cache -skip-name-resolve -datadir=/data/mysql -socket=/var/lib/mysql/mysql.sock -secure-file-priv=/var/lib/mysql-files -character-set-server=utf8 -user=mysql -symbolic-links=0 -pid-file=/var/run/mysqld/mysqld.pid -sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-web/Dockerfile b/km-dist/docker/knowstreaming-web/Dockerfile deleted file mode 100644 index 54b277db..00000000 --- a/km-dist/docker/knowstreaming-web/Dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -FROM docker.io/amd64/nginx:1.22 - -LABEL maintainer="haoqi123 <1148648445@qq.com>" - -WORKDIR /knowstreaming - -ARG VERSION_NODE=14.20.0 -ARG VERSION_KM=3.0.0-beta.2 - -# SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN apt update \ - && apt install -y git unzip - -RUN echo 'server { \ - listen 80; \ - server_name localhost; \ - gzip on; \ - gzip_buffers 16 8k; \ - gzip_comp_level 4; \ - gzip_http_version 1.0; \ - gzip_min_length 1280; \ - gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; \ - gzip_vary on; \ - root /pub; \ - location / { \ - root /pub; \ - if ($request_filename ~* .*\.(?:htm|html|json)$) { \ - add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; \ - } \ - try_files $uri /layout/index.html; \ - } \ - location ~* \.(json)$ { \ - add_header Cache-Control no-cache; \ - } \ - location @kmfallback { \ - } \ - #location ~ ^/(clusters|config|cluster|login) { \ - # rewrite ^.*$ /; \ - #} \ - location ~ ^/ks-km/api/v3 { \ - #rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; \ - proxy_pass http://knowstreaming-manager; \ - #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \ - #proxy_cookie_path /ks-km/ /; \ - #proxy_set_header Host $host; \ - #proxy_set_header Referer $http_referer; \ - #proxy_set_header Cookie $http_cookie; \ - #proxy_set_header X-Real-Ip $remote_addr; \ - #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; \ - } \ - location ~ ^/logi-security/api/v1 { \ - #rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; \ - proxy_pass http://knowstreaming-manager; \ - #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \ - } \ - location ~ ^/(401|403|404|500){ \ - rewrite ^.*$ /; \ - } \ - }' > /etc/nginx/conf.d/default.conf - -RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/v${VERSION_NODE}/node-v${VERSION_NODE}-linux-x64.tar.gz" \ - && mkdir node/ \ - && tar -zxf "node-v${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \ - && rm -rf /knowstreaming/*.tar.gz \ - && ln -s /knowstreaming/node/bin/node /usr/local/bin/node \ - && ln -s /knowstreaming/node/bin/npm /usr/local/bin/npm \ - && npm config set strict-ssl false \ - && npm config set registry https://registry.npm.taobao.org \ - && npm info underscore \ - && node --version \ - && npm --version \ - && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/v${VERSION_KM}.zip \ - && unzip -o -q v${VERSION_KM}.zip \ - && cd KnowStreaming-${VERSION_KM}/km-console \ - && npm install -g lerna && npm run i && npm run build \ - && mv /knowstreaming/KnowStreaming-${VERSION_KM}/km-rest/src/main/resources/templates /pub \ - && npm uninstall -g lerna \ - && cd / \ - && rm -rf /knowstreaming/* /usr/local/bin/node /usr/local/bin/npm \ - && apt remove -y git unzip && apt autoremove -y && apt remove --purge --auto-remove -y \ - && rm -rf /var/lib/apt/lists/* /tmp /root/.npm - -#ENTRYPOINT ["ls", "/ks_ui/km-console"] - -#CMD ["nginx", "-g", "daemon off;"]