1. 添加docker-compose部署方式

2. 更改manage服务初始化方式 3. 更改es初始化方式
2025-12-24 11:52:08 +08:00 · 2022-09-15 14:26:45 +08:00
parent 61efdf492f
commit 0c14c641d0
7 changed files with 157 additions and 57 deletions
--- a/km-dist/docker/docker-compose.yml
+++ b/km-dist/docker/docker-compose.yml
@@ -15,31 +15,17 @@ services:
      - kmysql
    expose:
      - 80
-    #ports:
-    #  - '80:80'
    environment:
      TZ: Asia/Shanghai
-      server.port: 80
-      spring.datasource.know-streaming.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
-      spring.datasource.know-streaming.username: root
-      spring.datasource.know-streaming.password: admin2022_

-      spring.logi-job.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
-      spring.logi-job.username: root
-      spring.logi-job.password: admin2022_
+      SERVER_MYSQL_ADDRESS: kmysql:3306
+      SERVER_MYSQL_DB: know_streaming
+      SERVER_MYSQL_USER: root
+      SERVER_MYSQL_PASSWORD: admin2022_

-      spring.logi-security.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
-      spring.logi-security.username: root
-      spring.logi-security.password: admin2022_
-      spring.logi-security.resource-extend-bean-name: myResourceExtendImpl
-      client-pool.kafka-consumer.max-total-client-num: 20
-
-      es.client.address: elasticsearch_single:9200
-      ES_CLIENT_ADDRESS: elasticsearch_single:9200
-      log.path: /logs
+      SERVER_ES_ADDRESS: elasticsearch_single:9200

      JAVA_OPTS: -Xmx1g -Xms1g -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08
-
    networks:
      - ks
    volumes:
@@ -63,15 +49,20 @@ services:
  elasticsearch_single:
    image: docker.io/library/elasticsearch:7.6.2
    container_name: elasticsearch_single
-    ports:
-      - '9200:9200'
-      - '9300:9300'
+    restart: always
+    expose:
+      - 9200
+      - 9300
+#    对外访问
+#    ports:
+#      - '9200:9200'
+#      - '9300:9300'
    environment:
      TZ: Asia/Shanghai
      ES_JAVA_OPTS: -Xms512m -Xmx512m
      discovery.type: single-node
-    volumes:
-      - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data
+#    volumes:
+#      - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data
    networks:
      - ks

@@ -79,6 +70,7 @@ services:
  kmysql:
    image: ks_mysql:1.0
    container_name: kmysql
+    restart: always
    environment:
      TZ: Asia/Shanghai
      MYSQL_ROOT_PASSWORD: admin2022_
--- a/km-dist/docker/knowstreaming-es/Dockerfile
+++ b/km-dist/docker/knowstreaming-es/Dockerfile
@@ -1,6 +0,0 @@
-FROM docker.io/library/elasticsearch:7.6.2
-LABEL author=haoqi6677
-#EXPOSE 9200
-#EXPOSE 9300
-#ARG INIT_SOURCE_FILE=./esinit/*
-#COPY ${INIT_SOURCE_FILE} /
--- a/km-dist/docker/knowstreaming-manage/Dockerfile
+++ b/km-dist/docker/knowstreaming-manage/Dockerfile
@@ -1,10 +1,11 @@
 FROM bitnami/java:11
-ARG JAVA_OPTS
-ARG JAR_FILE=./jar/*.jar
-ARG INIT_ES_FILE=./esinit/*
+ENV JAR_FILE=./jar/*.jar
+ENV INIT_ES_FILE=./esinit/*
+ENV CONFIG_FILE=./conf/*
 COPY ${JAR_FILE} /app/app.jar
 COPY ${INIT_ES_FILE} /
+COPY ${CONFIG_FILE} /conf/

 ENTRYPOINT ["/bin/bash","/es.sh"]
-CMD ["dumb-init","java","${JAVA_OPTS}", "-jar", "/app/app.jar"]
+CMD ["/bin/bash","/es.sh"]
 #ENTRYPOINT java -jar /app/app.jar
--- a/km-dist/docker/knowstreaming-manage/conf/application.yml
+++ b/km-dist/docker/knowstreaming-manage/conf/application.yml
@@ -0,0 +1,107 @@
+server:
+  port: 80 # 服务端口
+  tomcat:
+    accept-count: 1000
+    max-connections: 10000
+
+spring:
+  application:
+    name: know-streaming
+  profiles:
+    active: dev
+  main:
+    allow-bean-definition-overriding: true
+  jackson:
+    time-zone: GMT+8
+  datasource:
+    know-streaming: # know-streaming 自身数据库的配置
+      jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
+      username: SERVER_MYSQL_USER
+      password: SERVER_MYSQL_PASSWORD
+      driver-class-name: org.mariadb.jdbc.Driver
+      maximum-pool-size: 20
+      idle-timeout: 30000
+      connection-test-query: SELECT 1
+  logi-job:  # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
+    jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
+    username: SERVER_MYSQL_USER
+    password: SERVER_MYSQL_PASSWORD
+    driver-class-name: org.mariadb.jdbc.Driver
+    max-lifetime: 60000
+    init-sql: true
+    init-thread-num: 20
+    max-thread-num: 50
+    log-expire: 3 # 日志保存天数，以天为单位
+    app-name: know-streaming
+    claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual
+  logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
+    jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
+    username: SERVER_MYSQL_USER
+    password: SERVER_MYSQL_PASSWORD
+    driver-class-name: org.mariadb.jdbc.Driver
+    app-name: know-streaming
+    resource-extend-bean-name: myResourceExtendImpl
+    login-extend-bean-name: logiSecurityDefaultLoginExtendImpl
+
+logging:
+  config: classpath:logback-spring.xml
+
+# 线程池大小相关配置
+thread-pool:
+  scheduled:
+    thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
+
+  collector:                        # 采集模块的配置
+    future-util:                    # 采集模块线程池配置
+      num: 3                        # 线程池个数
+      thread-num: 64                # 每个线程池核心线程数
+      queue-size: 10000             # 每个线程池队列大小
+      select-suitable-enable: true  # 任务是否自动选择合适的线程池，非主要，可不修改
+      suitable-queue-size: 1000     # 线程池理想的队列大小，非主要，可不修改
+
+  task:                             # 任务模块的配置
+    metrics:                        # metrics采集任务配置
+      thread-num: 18                # metrics采集任务线程池核心线程数
+      queue-size: 180               # metrics采集任务线程池队列大小
+    metadata:                       # metadata同步任务配置
+      thread-num: 27                # metadata同步任务线程池核心线程数
+      queue-size: 270               # metadata同步任务线程池队列大小
+    common:                         # 剩余其他任务配置
+      thread-num: 15                # 剩余其他任务线程池核心线程数
+      queue-size: 150               # 剩余其他任务线程池队列大小
+
+
+# 客户端池大小相关配置
+client-pool:
+  kafka-consumer:
+    min-idle-client-num: 2        # 最小空闲客户端数
+    max-idle-client-num: 20       # 最大空闲客户端数
+    max-total-client-num: 20      # 最大客户端数
+    borrow-timeout-unit-ms: 5000  # 租借超时时间，单位秒
+
+
+# ES客户端配置
+es:
+  client:
+    address: SERVER_ES_ADDRESS
+    client-cnt: 10
+    io-thread-cnt: 2
+    max-retry-cnt: 5
+
+# 普罗米修斯指标导出相关配置
+management:
+  endpoints:
+    web:
+      base-path: /metrics
+      exposure:
+        include: '*'
+  health:
+    elasticsearch:
+      enabled: false
+  metrics:
+    export:
+      prometheus:
+        descriptions: true
+        enabled: true
+    tags:
+      application: know-streaming
--- a/km-dist/docker/knowstreaming-manage/esinit/es.sh
+++ b/km-dist/docker/knowstreaming-manage/esinit/es.sh
@@ -1,2 +1,9 @@
 /bin/bash /template.sh start
-java ${JAVA_OPTS} -jar /app/app.jar
+
+sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml
+sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml
+sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml
+sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml
+sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml
+
+java ${JAVA_OPTS} -jar /app/app.jar --spring.config.location=/conf/application.yml
--- a/km-dist/docker/knowstreaming-manage/esinit/template.sh
+++ b/km-dist/docker/knowstreaming-manage/esinit/template.sh
@@ -1,17 +1,16 @@
-esAddrPort=${ES_CLIENT_ADDRESS}
 echo "Wait ElasticSearch Start..."
 while true
 do
-  curl -s --connect-timeout 10 -o /dev/null  http://${esAddrPort}/_cat/nodes >> /dev/null 2>&1
+  curl -s --connect-timeout 10 -o /dev/null  http://${SERVER_ES_ADDRESS}/_cat/nodes >> /dev/null 2>&1
  if [ "$?" != "0" ];then
    sleep 1s
  else
-    echo "ElasticSearch Start Success"
+    echo "ElasticSearch Start Initialize"
    break
  fi
 done

-curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_broker_metric -d '{
+curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{
    "order" : 10,
    "index_patterns" : [
      "ks_kafka_broker_metric*"
@@ -113,7 +112,7 @@ curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -
    "aliases" : { }
  }' 

-curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_cluster_metric -d '{
+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{
    "order" : 10,
    "index_patterns" : [
      "ks_kafka_cluster_metric*"
@@ -298,9 +297,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
      }
    },
    "aliases" : { }
-   }' 
+   }'

-curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_group_metric -d '{
+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{
    "order" : 10,
    "index_patterns" : [
      "ks_kafka_group_metric*"
@@ -373,9 +372,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
      }
    },
    "aliases" : { }
-  }' 
+  }'

-curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_partition_metric -d '{
+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{
    "order" : 10,
    "index_patterns" : [
      "ks_kafka_partition_metric*"
@@ -439,9 +438,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
      }
    },
    "aliases" : { }
-  }' 
+  }'

-curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_replication_metric -d '{
+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{
    "order" : 10,
    "index_patterns" : [
      "ks_kafka_partition_metric*"
@@ -528,9 +527,9 @@ PUT _template/ks_kafka_replication_metric
      }
    },
    "aliases" : { }
-  }' 
+  }'

-curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_topic_metric -d '{
+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{
    "order" : 10,
    "index_patterns" : [
      "ks_kafka_topic_metric*"
@@ -645,17 +644,17 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
      }
    },
    "aliases" : { }
-  }' 
+  }'

-for i in `seq 6`;
+for i in {0..6};
 do
    logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
-    curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_broker_metric${logdate}  && \
-    curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_cluster_metric${logdate} && \
-    curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_group_metric${logdate} && \
-    curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_partition_metric${logdate} && \
-    curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_replication_metric${logdate} && \
-    curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_topic_metric${logdate} || \
+    curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate}  && \
+    curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \
+    curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \
+    curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \
+    curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \
+    curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \
    exit 2
 done

--- a/km-dist/docker/knowstreaming-mysql/Dockerfile
+++ b/km-dist/docker/knowstreaming-mysql/Dockerfile
@@ -1,6 +1,6 @@
 FROM mysql:5.7.39-oracle
 EXPOSE 3306
-ARG INIT_SOURCE_FILE=./sql/*.sql
+ENV INIT_SOURCE_FILE=./sql/*.sql
 #RUN mkdir /data/mysql
 #RUN mkdir /var/lib/mysql-files
 COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/