8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 03:42:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Optimize]优化 MySQL & ES 测试容器的初始化(#906)

Browse Source 
主要的变更
1、knowstreaming/knowstreaming-manager 容器；
2、knowstreaming/knowstreaming-mysql 容器调整为使用 mysql:5.7 容器；
3、初始化 mysql:5.7 容器后，增加初始化 MySQL 表及数据的动作；

被影响的变更：
1、移动 km-dist/init/sql 下的MySQL初始化脚本至 km-persistence/src/main/resource/sql 下，以便项目测试时加载到所需的初始化 SQL；
2、删除无用的 km-dist/init/template 目录；
3、因为 km-dist/init/sql 和 km-dist/init/template 目录的调整，因此也调整 ReleaseKnowStreaming.xml 内的文件内容；
This commit is contained in:
zengqiao
2023-02-10 17:01:33 +08:00
committed by EricZeng
parent c062586c7e
commit b5683b73c2
19 changed files with 148 additions and 925 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
km-dist/ReleaseKnowStreaming.xml
View File

@@ -26,6 +26,22 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>../km-persistence/src/main/resources</directory>
<outputDirectory>init</outputDirectory>
<includes>
<include>sql</include>
<include>sql/*</include>
</includes>
</fileSet>
<fileSet>
<directory>../km-persistence/src/main/resources/es</directory>
<outputDirectory>init</outputDirectory>
<includes>
<include>template</include>
<include>template/*</include>
</includes>
</fileSet>
<fileSet>
<includes>
@@ -34,8 +50,6 @@
<include>init/*</include>
<include>init/*/*</include>
<include>packages/*</include>
</includes>
</fileSet>
</fileSets>

101
km-dist/init/template/ks_kafka_broker_metric
View File

@@ -1,101 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_broker_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "2"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"NetworkProcessorAvgIdle" : {
"type" : "float"
},
"UnderReplicatedPartitions" : {
"type" : "float"
},
"BytesIn_min_15" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"RequestHandlerAvgIdle" : {
"type" : "float"
},
"connectionsCount" : {
"type" : "float"
},
"BytesIn_min_5" : {
"type" : "float"
},
"HealthScore" : {
"type" : "float"
},
"BytesOut" : {
"type" : "float"
},
"BytesOut_min_15" : {
"type" : "float"
},
"BytesIn" : {
"type" : "float"
},
"BytesOut_min_5" : {
"type" : "float"
},
"TotalRequestQueueSize" : {
"type" : "float"
},
"MessagesIn" : {
"type" : "float"
},
"TotalProduceRequests" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
},
"TotalResponseQueueSize" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}

186
km-dist/init/template/ks_kafka_cluster_metric
View File

@@ -1,186 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_cluster_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "2"
}
},
"mappings" : {
"properties" : {
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"Connections" : {
"type" : "double"
},
"BytesIn_min_15" : {
"type" : "double"
},
"PartitionURP" : {
"type" : "double"
},
"HealthScore_Topics" : {
"type" : "double"
},
"EventQueueSize" : {
"type" : "double"
},
"ActiveControllerCount" : {
"type" : "double"
},
"GroupDeads" : {
"type" : "double"
},
"BytesIn_min_5" : {
"type" : "double"
},
"HealthCheckTotal_Topics" : {
"type" : "double"
},
"Partitions" : {
"type" : "double"
},
"BytesOut" : {
"type" : "double"
},
"Groups" : {
"type" : "double"
},
"BytesOut_min_15" : {
"type" : "double"
},
"TotalRequestQueueSize" : {
"type" : "double"
},
"HealthCheckPassed_Groups" : {
"type" : "double"
},
"TotalProduceRequests" : {
"type" : "double"
},
"HealthCheckPassed" : {
"type" : "double"
},
"TotalLogSize" : {
"type" : "double"
},
"GroupEmptys" : {
"type" : "double"
},
"PartitionNoLeader" : {
"type" : "double"
},
"HealthScore_Brokers" : {
"type" : "double"
},
"Messages" : {
"type" : "double"
},
"Topics" : {
"type" : "double"
},
"PartitionMinISR_E" : {
"type" : "double"
},
"HealthCheckTotal" : {
"type" : "double"
},
"Brokers" : {
"type" : "double"
},
"Replicas" : {
"type" : "double"
},
"HealthCheckTotal_Groups" : {
"type" : "double"
},
"GroupRebalances" : {
"type" : "double"
},
"MessageIn" : {
"type" : "double"
},
"HealthScore" : {
"type" : "double"
},
"HealthCheckPassed_Topics" : {
"type" : "double"
},
"HealthCheckTotal_Brokers" : {
"type" : "double"
},
"PartitionMinISR_S" : {
"type" : "double"
},
"BytesIn" : {
"type" : "double"
},
"BytesOut_min_5" : {
"type" : "double"
},
"GroupActives" : {
"type" : "double"
},
"MessagesIn" : {
"type" : "double"
},
"GroupReBalances" : {
"type" : "double"
},
"HealthCheckPassed_Brokers" : {
"type" : "double"
},
"HealthScore_Groups" : {
"type" : "double"
},
"TotalResponseQueueSize" : {
"type" : "double"
},
"Zookeepers" : {
"type" : "double"
},
"LeaderMessages" : {
"type" : "double"
},
"HealthScore_Cluster" : {
"type" : "double"
},
"HealthCheckPassed_Cluster" : {
"type" : "double"
},
"HealthCheckTotal_Cluster" : {
"type" : "double"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"type" : "date"
}
}
},
"aliases" : { }
}

86
km-dist/init/template/ks_kafka_connect_cluster_metric
View File

@@ -1,86 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_connect_cluster_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "2"
}
},
"mappings" : {
"properties" : {
"connectClusterId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"ConnectorCount" : {
"type" : "float"
},
"TaskCount" : {
"type" : "float"
},
"ConnectorStartupAttemptsTotal" : {
"type" : "float"
},
"ConnectorStartupFailurePercentage" : {
"type" : "float"
},
"ConnectorStartupFailureTotal" : {
"type" : "float"
},
"ConnectorStartupSuccessPercentage" : {
"type" : "float"
},
"ConnectorStartupSuccessTotal" : {
"type" : "float"
},
"TaskStartupAttemptsTotal" : {
"type" : "float"
},
"TaskStartupFailurePercentage" : {
"type" : "float"
},
"TaskStartupFailureTotal" : {
"type" : "float"
},
"TaskStartupSuccessPercentage" : {
"type" : "float"
},
"TaskStartupSuccessTotal" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}

194
km-dist/init/template/ks_kafka_connect_connector_metric
View File

@@ -1,194 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_connect_connector_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "2"
}
},
"mappings" : {
"properties" : {
"connectClusterId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"connectorName" : {
"type" : "keyword"
},
"connectorNameAndClusterId" : {
"type" : "keyword"
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"HealthState" : {
"type" : "float"
},
"ConnectorTotalTaskCount" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"ConnectorRunningTaskCount" : {
"type" : "float"
},
"ConnectorPausedTaskCount" : {
"type" : "float"
},
"ConnectorFailedTaskCount" : {
"type" : "float"
},
"ConnectorUnassignedTaskCount" : {
"type" : "float"
},
"BatchSizeAvg" : {
"type" : "float"
},
"BatchSizeMax" : {
"type" : "float"
},
"OffsetCommitAvgTimeMs" : {
"type" : "float"
},
"OffsetCommitMaxTimeMs" : {
"type" : "float"
},
"OffsetCommitFailurePercentage" : {
"type" : "float"
},
"OffsetCommitSuccessPercentage" : {
"type" : "float"
},
"PollBatchAvgTimeMs" : {
"type" : "float"
},
"PollBatchMaxTimeMs" : {
"type" : "float"
},
"SourceRecordActiveCount" : {
"type" : "float"
},
"SourceRecordActiveCountAvg" : {
"type" : "float"
},
"SourceRecordActiveCountMax" : {
"type" : "float"
},
"SourceRecordPollRate" : {
"type" : "float"
},
"SourceRecordPollTotal" : {
"type" : "float"
},
"SourceRecordWriteRate" : {
"type" : "float"
},
"SourceRecordWriteTotal" : {
"type" : "float"
},
"OffsetCommitCompletionRate" : {
"type" : "float"
},
"OffsetCommitCompletionTotal" : {
"type" : "float"
},
"OffsetCommitSkipRate" : {
"type" : "float"
},
"OffsetCommitSkipTotal" : {
"type" : "float"
},
"PartitionCount" : {
"type" : "float"
},
"PutBatchAvgTimeMs" : {
"type" : "float"
},
"PutBatchMaxTimeMs" : {
"type" : "float"
},
"SinkRecordActiveCount" : {
"type" : "float"
},
"SinkRecordActiveCountAvg" : {
"type" : "float"
},
"SinkRecordActiveCountMax" : {
"type" : "float"
},
"SinkRecordLagMax" : {
"type" : "float"
},
"SinkRecordReadRate" : {
"type" : "float"
},
"SinkRecordReadTotal" : {
"type" : "float"
},
"SinkRecordSendRate" : {
"type" : "float"
},
"SinkRecordSendTotal" : {
"type" : "float"
},
"DeadletterqueueProduceFailures" : {
"type" : "float"
},
"DeadletterqueueProduceRequests" : {
"type" : "float"
},
"LastErrorTimestamp" : {
"type" : "float"
},
"TotalErrorsLogged" : {
"type" : "float"
},
"TotalRecordErrors" : {
"type" : "float"
},
"TotalRecordFailures" : {
"type" : "float"
},
"TotalRecordsSkipped" : {
"type" : "float"
},
"TotalRetries" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}

74
km-dist/init/template/ks_kafka_group_metric
View File

@@ -1,74 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_group_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "6"
}
},
"mappings" : {
"properties" : {
"group" : {
"type" : "keyword"
},
"partitionId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"topic" : {
"type" : "keyword"
},
"metrics" : {
"properties" : {
"HealthScore" : {
"type" : "float"
},
"Lag" : {
"type" : "float"
},
"OffsetConsumed" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
}
}
},
"groupMetric" : {
"type" : "keyword"
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}

65
km-dist/init/template/ks_kafka_partition_metric
View File

@@ -1,65 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_partition_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "6"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"partitionId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"topic" : {
"type" : "keyword"
},
"metrics" : {
"properties" : {
"LogStartOffset" : {
"type" : "float"
},
"Messages" : {
"type" : "float"
},
"LogEndOffset" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}

116
km-dist/init/template/ks_kafka_topic_metric
View File

@@ -1,116 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_topic_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "6"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"topic" : {
"type" : "keyword"
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"BytesIn_min_15" : {
"type" : "float"
},
"Messages" : {
"type" : "float"
},
"BytesRejected" : {
"type" : "float"
},
"PartitionURP" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"ReplicationCount" : {
"type" : "float"
},
"ReplicationBytesOut" : {
"type" : "float"
},
"ReplicationBytesIn" : {
"type" : "float"
},
"FailedFetchRequests" : {
"type" : "float"
},
"BytesIn_min_5" : {
"type" : "float"
},
"HealthScore" : {
"type" : "float"
},
"LogSize" : {
"type" : "float"
},
"BytesOut" : {
"type" : "float"
},
"BytesOut_min_15" : {
"type" : "float"
},
"FailedProduceRequests" : {
"type" : "float"
},
"BytesIn" : {
"type" : "float"
},
"BytesOut_min_5" : {
"type" : "float"
},
"MessagesIn" : {
"type" : "float"
},
"TotalProduceRequests" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
}
}
},
"brokerAgg" : {
"type" : "keyword"
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}

84
km-dist/init/template/ks_kafka_zookeeper_metric
View File

@@ -1,84 +0,0 @@
{
"order" : 10,
"index_patterns" : [
"ks_kafka_zookeeper_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "2"
}
},
"mappings" : {
"properties" : {
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"AvgRequestLatency" : {
"type" : "double"
},
"MinRequestLatency" : {
"type" : "double"
},
"MaxRequestLatency" : {
"type" : "double"
},
"OutstandingRequests" : {
"type" : "double"
},
"NodeCount" : {
"type" : "double"
},
"WatchCount" : {
"type" : "double"
},
"NumAliveConnections" : {
"type" : "double"
},
"PacketsReceived" : {
"type" : "double"
},
"PacketsSent" : {
"type" : "double"
},
"EphemeralsCount" : {
"type" : "double"
},
"ApproximateDataSize" : {
"type" : "double"
},
"OpenFileDescriptorCount" : {
"type" : "double"
},
"MaxFileDescriptorCount" : {
"type" : "double"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"type" : "date"
}
}
},
"aliases" : { }
}

55
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/utils/LoadSQLUtil.java Normal file
View File

@@ -0,0 +1,55 @@
package com.xiaojukeji.know.streaming.km.persistence.utils;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
public class LoadSQLUtil {
private static final ILog LOGGER = LogFactory.getLog(LoadSQLUtil.class);
public static final String SQL_DDL_KS_KM = "sql/ddl-ks-km.sql";
public static final String SQL_DDL_LOGI_JOB = "sql/ddl-logi-job.sql";
public static final String SQL_DDL_LOGI_SECURITY = "sql/ddl-logi-security.sql";
public static final String SQL_DML_KS_KM = "sql/dml-ks-km.sql";
public static final String SQL_DML_LOGI = "sql/dml-logi.sql";
public static String loadSQL(String sqlFileName) {
InputStream inputStream = LoadSQLUtil.class.getClassLoader().getResourceAsStream(sqlFileName);
if (inputStream == null) {
LOGGER.error("method=loadSQL||fileName={}||msg=read script failed", sqlFileName);
return "";
}
try {
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
String line = null;
StringBuilder sb = new StringBuilder();
while ((line = bufferedReader.readLine()) != null) {
sb.append(line).append("\n");
}
return sb.toString();
} catch (Exception e) {
LOGGER.error("method=loadSQL||fileName={}||errMsg={}||msg=read script failed", sqlFileName, e.getMessage());
} finally {
try {
inputStream.close();
} catch (IOException e) {
LOGGER.error("method=loadSQL||fileName={}||errMsg={}||msg=close reading script failed", sqlFileName, e.getMessage());
}
}
return "";
}
private LoadSQLUtil() {
}
}

0
km-dist/init/sql/ddl-ks-km.sql → km-persistence/src/main/resources/sql/ddl-ks-km.sql
View File

0
km-dist/init/sql/ddl-logi-job.sql → km-persistence/src/main/resources/sql/ddl-logi-job.sql
View File

0
km-dist/init/sql/ddl-logi-security.sql → km-persistence/src/main/resources/sql/ddl-logi-security.sql
View File

0
km-dist/init/sql/dml-ks-km.sql → km-persistence/src/main/resources/sql/dml-ks-km.sql
View File

0
km-dist/init/sql/dml-logi.sql → km-persistence/src/main/resources/sql/dml-logi.sql
View File

8
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/KMTestEnvService.java
View File

@@ -42,8 +42,16 @@ public abstract class KMTestEnvService {
@DynamicPropertySource
static void setUp(DynamicPropertyRegistry registry) {
registry.add("spring.datasource.know-streaming.jdbc-url", mySQLTestContainer.jdbcUrl());
registry.add("spring.datasource.know-streaming.username", mySQLTestContainer.jdbcUsername());
registry.add("spring.datasource.know-streaming.password", mySQLTestContainer.jdbcPassword());
registry.add("spring.logi-job.jdbc-url", mySQLTestContainer.jdbcUrl());
registry.add("spring.logi-job.username", mySQLTestContainer.jdbcUsername());
registry.add("spring.logi-job.password", mySQLTestContainer.jdbcPassword());
registry.add("spring.logi-security.jdbc-url", mySQLTestContainer.jdbcUrl());
registry.add("spring.logi-security.username", mySQLTestContainer.jdbcUsername());
registry.add("spring.logi-security.password", mySQLTestContainer.jdbcPassword());
registry.add("es.client.address", esTestContainer.esUrl());
}

11
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/es/ESTestContainer.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.test.container.es;
import com.xiaojukeji.know.streaming.test.container.BaseTestContainer;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
@@ -19,14 +18,6 @@ public class ESTestContainer extends BaseTestContainer {
.withEnv("ES_JAVA_OPTS", "-Xms512m -Xmx512m")
.withEnv("discovery.type", "single-node");
// km容器，需要初始化es索引模版
private static final GenericContainer<?> INIT_CONTAINER = new GenericContainer<>(
"knowstreaming/knowstreaming-manager:latest"
)
.withEnv("TZ", "Asia/Shanghai")
.withCommand("/bin/bash", "/es_template_create.sh")
.dependsOn(ES_CONTAINER);
@NotNull
public Supplier<Object> esUrl() {
return () -> ES_CONTAINER.getHost() + ":" + ES_CONTAINER.getMappedPort(9200);
@@ -34,7 +25,7 @@ public class ESTestContainer extends BaseTestContainer {
@Override
public void init() {
Startables.deepStart(ES_CONTAINER, INIT_CONTAINER).join();
Startables.deepStart(ES_CONTAINER).join();
}
@Override

40
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/KSMySQLContainer.java
View File

@@ -1,11 +1,20 @@
package com.xiaojukeji.know.streaming.test.container.mysql;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.containers.ContainerLaunchException;
import org.testcontainers.containers.JdbcDatabaseContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.delegate.DatabaseDelegate;
import org.testcontainers.ext.ScriptUtils;
import org.testcontainers.jdbc.JdbcDatabaseDelegate;
import org.testcontainers.utility.DockerImageName;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
@@ -13,6 +22,8 @@ import java.util.Set;
* @see org.testcontainers.containers.MySQLContainer
*/
public class KSMySQLContainer<SELF extends KSMySQLContainer<SELF>> extends JdbcDatabaseContainer<SELF> {
private static final ILog LOGGER = LogFactory.getLog(KSMySQLContainer.class);
public static final String NAME = "mysql";
private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("mysql");
@@ -39,6 +50,8 @@ public class KSMySQLContainer<SELF extends KSMySQLContainer<SELF>> extends JdbcD
private static final String MYSQL_ROOT_USER = "root";
private List<Tuple<String, String>> initScriptPathAndContentList = new ArrayList<>();
/**
* @deprecated use {@link MySQLContainer(DockerImageName)} instead
*/
@@ -169,4 +182,31 @@ public class KSMySQLContainer<SELF extends KSMySQLContainer<SELF>> extends JdbcD
this.password = password;
return self();
}
public SELF addInitScriptPathAndContent(String initScriptPath, String initScriptContent) {
initScriptPathAndContentList.add(new Tuple<>(initScriptPath, initScriptContent));
return self();
}
// KS改动的地方
@Override
public DatabaseDelegate getDatabaseDelegate() {
return new JdbcDatabaseDelegate(this, "");
}
@Override
protected void runInitScriptIfRequired() {
if (initScriptPathAndContentList.isEmpty()) {
return;
}
for (Tuple<String, String> elem: initScriptPathAndContentList) {
try {
ScriptUtils.executeDatabaseScript(this.getDatabaseDelegate(), elem.getV1(), elem.getV2());
} catch (ScriptException var5) {
LOGGER.error("Error while executing init script: {}", elem.getV1(), var5);
throw new ScriptUtils.UncategorizedScriptException("Error while executing init script: " + elem.getV2(), var5);
}
}
}
}

35
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/MySQLTestContainer.java
View File

@@ -1,5 +1,6 @@
package com.xiaojukeji.know.streaming.test.container.mysql;
import com.xiaojukeji.know.streaming.km.persistence.utils.LoadSQLUtil;
import com.xiaojukeji.know.streaming.test.container.BaseTestContainer;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.lifecycle.Startables;
@@ -8,7 +9,12 @@ import org.testcontainers.utility.DockerImageName;
import java.util.function.Supplier;
public class MySQLTestContainer extends BaseTestContainer {
private static final String DB_PROPERTY = "?useUnicode=true" +
private static final String DB_USERNAME = "root";
private static final String DB_PASSWORD = "1234567890";
private static final String DATABASE_NAME = "know_streaming";
private static final String DB_PROPERTY = "?useUnicode=true" +
"&characterEncoding=utf8" +
"&jdbcCompliantTruncation=true" +
"&allowMultiQueries=true" +
@@ -18,19 +24,34 @@ public class MySQLTestContainer extends BaseTestContainer {
"&allowPublicKeyRetrieval=true";
private static final KSMySQLContainer<?> MYSQL_CONTAINER = new KSMySQLContainer<>(
DockerImageName.parse("knowstreaming/knowstreaming-mysql:latest").asCompatibleSubstituteFor("mysql")
DockerImageName.parse("mysql:5.7").asCompatibleSubstituteFor("mysql")
)
.withEnv("MYSQL_ROOT_HOST", "%")
.withEnv("TZ", "Asia/Shanghai")
.withDatabaseName("know_streaming")
.withUsername("root")
.withPassword("mysql_pass");
.withDatabaseName(DATABASE_NAME)
.withUsername(DB_USERNAME)
.withPassword(DB_PASSWORD)
.addInitScriptPathAndContent(LoadSQLUtil.SQL_DDL_KS_KM, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DDL_KS_KM)))
.addInitScriptPathAndContent(LoadSQLUtil.SQL_DDL_LOGI_JOB, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DDL_LOGI_JOB)))
.addInitScriptPathAndContent(LoadSQLUtil.SQL_DDL_LOGI_SECURITY, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DDL_LOGI_SECURITY)))
.addInitScriptPathAndContent(LoadSQLUtil.SQL_DML_KS_KM, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DML_KS_KM)))
.addInitScriptPathAndContent(LoadSQLUtil.SQL_DML_LOGI, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DML_LOGI)))
;
@NotNull
public Supplier<Object> jdbcUsername() {
return () -> DB_USERNAME;
}
@NotNull
public Supplier<Object> jdbcPassword() {
return () -> DB_PASSWORD;
}
@NotNull
public Supplier<Object> jdbcUrl() {
return () -> "jdbc:mariadb://"
+ MYSQL_CONTAINER.getHost() + ":" + MYSQL_CONTAINER.getMappedPort(3306)
+ "/know_streaming" + DB_PROPERTY;
+ "/" + DATABASE_NAME + DB_PROPERTY;
}
@Override