调整v3.2版本容器化部署信息

km-dist/docker/manager/application.yml

@@ -0,0 +1,119 @@
+server:
+  port: 80 # 服务端口
+  tomcat:
+    accept-count: 1000
+    max-connections: 10000
+
+spring:
+  application:
+    name: know-streaming
+  profiles:
+    active: dev
+  main:
+    allow-bean-definition-overriding: true
+  jackson:
+    time-zone: GMT+8
+  datasource:
+    know-streaming: # know-streaming 自身数据库的配置
+      jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
+      username: SERVER_MYSQL_USER
+      password: SERVER_MYSQL_PASSWORD
+      driver-class-name: org.mariadb.jdbc.Driver
+      maximum-pool-size: 20
+      idle-timeout: 30000
+      connection-test-query: SELECT 1
+  logi-job:  # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
+    enable: true 
+    jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
+    username: SERVER_MYSQL_USER
+    password: SERVER_MYSQL_PASSWORD
+    driver-class-name: org.mariadb.jdbc.Driver
+    max-lifetime: 60000
+    init-sql: true
+    init-thread-num: 20
+    max-thread-num: 50
+    log-expire: 3 # 日志保存天数，以天为单位
+    app-name: know-streaming
+    claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual
+  logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
+    jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
+    username: SERVER_MYSQL_USER
+    password: SERVER_MYSQL_PASSWORD
+    driver-class-name: org.mariadb.jdbc.Driver
+    app-name: know-streaming
+    resource-extend-bean-name: myResourceExtendImpl
+    login-extend-bean-name: logiSecurityDefaultLoginExtendImpl
+
+logging:
+  config: classpath:logback-spring.xml
+
+# 线程池大小相关配置
+thread-pool:
+  scheduled:
+    thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
+
+  collector:                        # 采集模块的配置
+    future-util:                    # 采集模块线程池配置
+      num: 3                        # 线程池个数
+      thread-num: 64                # 每个线程池核心线程数
+      queue-size: 10000             # 每个线程池队列大小
+      select-suitable-enable: true  # 任务是否自动选择合适的线程池，非主要，可不修改
+      suitable-queue-size: 1000     # 线程池理想的队列大小，非主要，可不修改
+
+  task:                             # 任务模块的配置
+    metrics:                        # metrics采集任务配置
+      thread-num: 18                # metrics采集任务线程池核心线程数
+      queue-size: 180               # metrics采集任务线程池队列大小
+    metadata:                       # metadata同步任务配置
+      thread-num: 27                # metadata同步任务线程池核心线程数
+      queue-size: 270               # metadata同步任务线程池队列大小
+    common:                         # 剩余其他任务配置
+      thread-num: 15                # 剩余其他任务线程池核心线程数
+      queue-size: 150               # 剩余其他任务线程池队列大小
+  es:
+    search:                         # es查询线程池
+      thread-num: 20                # 线程池大小
+      queue-size: 10000             # 队列大小
+
+
+
+# 客户端池大小相关配置
+client-pool:
+  kafka-consumer:
+    min-idle-client-num: 2        # 最小空闲客户端数
+    max-idle-client-num: 20       # 最大空闲客户端数
+    max-total-client-num: 20      # 最大客户端数
+    borrow-timeout-unit-ms: 5000  # 租借超时时间，单位秒
+  kafka-admin:
+    client-cnt: 1                 # 每个Kafka集群创建的KafkaAdminClient数
+
+
+
+# ES客户端配置
+es:
+  client:
+    address: SERVER_ES_ADDRESS
+    client-cnt: 10
+    io-thread-cnt: 2
+    max-retry-cnt: 5
+  index:
+    expire: 15                    # 索引过期天数，15表示超过15天的索引会被KS过期删除
+
+
+# 普罗米修斯指标导出相关配置
+management:
+  endpoints:
+    web:
+      base-path: /metrics
+      exposure:
+        include: '*'
+  health:
+    elasticsearch:
+      enabled: false
+  metrics:
+    export:
+      prometheus:
+        descriptions: true
+        enabled: true
+    tags:
+      application: know-streaming

km-dist/docker/manager/logback-spring.xml

@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration scan="true" scanPeriod="10 seconds">
+    <contextName>logback</contextName>
+    <property name="log.path" value="./logs" />
+
+    <!-- 彩色日志 -->
+    <!-- 彩色日志依赖的渲染类 -->
+    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
+    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
+    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
+    <!-- 彩色日志格式 -->
+    <property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
+
+    <!--输出到控制台-->
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>info</level>
+        </filter>
+        <encoder>
+            <Pattern>${CONSOLE_LOG_PATTERN}</Pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+
+    <!--输出到文件-->
+
+    <!-- 时间滚动输出 level为 DEBUG 日志 -->
+    <appender name="DEBUG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/log_debug.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 日志归档 -->
+            <fileNamePattern>${log.path}/log_debug_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>2</maxHistory>
+            <totalSizeCap>5GB</totalSizeCap>
+        </rollingPolicy>
+        <!-- 此日志文件只记录debug级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>debug</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 INFO 日志 -->
+    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${log.path}/log_info.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 每天日志归档路径以及格式 -->
+            <fileNamePattern>${log.path}/log_info_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录info级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>info</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 WARN 日志 -->
+    <appender name="WARN_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${log.path}/log_warn.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/log_warn_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录warn级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>warn</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+
+    <!-- 时间滚动输出 level为 ERROR 日志 -->
+    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${log.path}/log_error.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/log_error_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录ERROR级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ERROR</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- Metrics信息收集日志 -->
+    <appender name="ES_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/es/es.log</file>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/es/es_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>3</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <!-- Metrics信息收集日志 -->
+    <appender name="METRIC_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/metric/metrics.log</file>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/metric/metrics_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>3</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <!-- Task信息收集日志 -->
+    <appender name="TASK_LOGGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/task/task.log</file>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${log.path}/task/task_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>3</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <!-- level为 DEBUG 日志，时间滚动输出  -->
+    <appender name="logIJobLogger" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.path}/logIJob/logIJob.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <FileNamePattern>${log.path}/logIJob/logIJob.log.%d{yyyy-MM-dd}</FileNamePattern>
+            <maxHistory>${maxHistory}</maxHistory>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <logger name="ES_LOGGER" level="ERROR" additivity="false">
+        <appender-ref ref="ES_LOGGER"/>
+    </logger>
+
+    <logger name="METRIC_LOGGER" level="ERROR" additivity="false">
+        <appender-ref ref="METRIC_LOGGER"/>
+    </logger>
+
+    <logger name="TASK_LOGGER" level="ERROR" additivity="false">
+        <appender-ref ref="TASK_LOGGER"/>
+    </logger>
+
+    <logger name="com.didiglobal.logi.job" level="ERROR" additivity="false">
+        <appender-ref ref="logIJobLogger" />
+    </logger>
+
+    <logger name="org.apache.zookeeper" level="WARN" additivity="false" />
+    <logger name="org.apache.ibatis" level="INFO" additivity="false" />
+    <logger name="org.mybatis.spring" level="INFO" additivity="false" />
+    <logger name="com.github.miemiedev.mybatis.paginator" level="INFO" additivity="false" />
+
+    <root level="debug">
+        <appender-ref ref="CONSOLE" />
+        <appender-ref ref="DEBUG_FILE" />
+        <appender-ref ref="INFO_FILE" />
+        <appender-ref ref="WARN_FILE" />
+        <appender-ref ref="ERROR_FILE" />
+        <!--<appender-ref ref="METRICS_LOG" />-->
+    </root>
+
+    <!--生产环境:输出到文件-->
+    <!--<springProfile name="pro">-->
+    <!--<root level="info">-->
+    <!--<appender-ref ref="CONSOLE" />-->
+    <!--<appender-ref ref="DEBUG_FILE" />-->
+    <!--<appender-ref ref="INFO_FILE" />-->
+    <!--<appender-ref ref="ERROR_FILE" />-->
+    <!--<appender-ref ref="WARN_FILE" />-->
+    <!--</root>-->
+    <!--</springProfile>-->
+</configuration>