diff --git a/km-dist/ReleaseKnowStreaming.xml b/km-dist/ReleaseKnowStreaming.xml index 046a5cfd..37833d4b 100755 --- a/km-dist/ReleaseKnowStreaming.xml +++ b/km-dist/ReleaseKnowStreaming.xml @@ -26,6 +26,22 @@ 0755 + + ../km-persistence/src/main/resources + init + + sql + sql/* + + + + ../km-persistence/src/main/resources/es + init + + template + template/* + + @@ -34,8 +50,6 @@ init/* init/*/* - - packages/* diff --git a/km-dist/init/template/ks_kafka_broker_metric b/km-dist/init/template/ks_kafka_broker_metric deleted file mode 100644 index 749b1494..00000000 --- a/km-dist/init/template/ks_kafka_broker_metric +++ /dev/null @@ -1,101 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_broker_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "NetworkProcessorAvgIdle" : { - "type" : "float" - }, - "UnderReplicatedPartitions" : { - "type" : "float" - }, - "BytesIn_min_15" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "RequestHandlerAvgIdle" : { - "type" : "float" - }, - "connectionsCount" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "TotalRequestQueueSize" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "TotalResponseQueueSize" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_cluster_metric b/km-dist/init/template/ks_kafka_cluster_metric deleted file mode 100644 index 942fdce2..00000000 --- a/km-dist/init/template/ks_kafka_cluster_metric +++ /dev/null @@ -1,186 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "Connections" : { - "type" : "double" - }, - "BytesIn_min_15" : { - "type" : "double" - }, - "PartitionURP" : { - "type" : "double" - }, - "HealthScore_Topics" : { - "type" : "double" - }, - "EventQueueSize" : { - "type" : "double" - }, - "ActiveControllerCount" : { - "type" : "double" - }, - "GroupDeads" : { - "type" : "double" - }, - "BytesIn_min_5" : { - "type" : "double" - }, - "HealthCheckTotal_Topics" : { - "type" : "double" - }, - "Partitions" : { - "type" : "double" - }, - "BytesOut" : { - "type" : "double" - }, - "Groups" : { - "type" : "double" - }, - "BytesOut_min_15" : { - "type" : "double" - }, - "TotalRequestQueueSize" : { - "type" : "double" - }, - "HealthCheckPassed_Groups" : { - "type" : "double" - }, - "TotalProduceRequests" : { - "type" : "double" - }, - "HealthCheckPassed" : { - "type" : "double" - }, - "TotalLogSize" : { - "type" : "double" - }, - "GroupEmptys" : { - "type" : "double" - }, - "PartitionNoLeader" : { - "type" : "double" - }, - "HealthScore_Brokers" : { - "type" : "double" - }, - "Messages" : { - "type" : "double" - }, - "Topics" : { - "type" : "double" - }, - "PartitionMinISR_E" : { - "type" : "double" - }, - "HealthCheckTotal" : { - "type" : "double" - }, - "Brokers" : { - "type" : "double" - }, - "Replicas" : { - "type" : "double" - }, - "HealthCheckTotal_Groups" : { - "type" : "double" - }, - "GroupRebalances" : { - "type" : "double" - }, - "MessageIn" : { - "type" : "double" - }, - "HealthScore" : { - "type" : "double" - }, - "HealthCheckPassed_Topics" : { - "type" : "double" - }, - "HealthCheckTotal_Brokers" : { - "type" : "double" - }, - "PartitionMinISR_S" : { - "type" : "double" - }, - "BytesIn" : { - "type" : "double" - }, - "BytesOut_min_5" : { - "type" : "double" - }, - "GroupActives" : { - "type" : "double" - }, - "MessagesIn" : { - "type" : "double" - }, - "GroupReBalances" : { - "type" : "double" - }, - "HealthCheckPassed_Brokers" : { - "type" : "double" - }, - "HealthScore_Groups" : { - "type" : "double" - }, - "TotalResponseQueueSize" : { - "type" : "double" - }, - "Zookeepers" : { - "type" : "double" - }, - "LeaderMessages" : { - "type" : "double" - }, - "HealthScore_Cluster" : { - "type" : "double" - }, - "HealthCheckPassed_Cluster" : { - "type" : "double" - }, - "HealthCheckTotal_Cluster" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_connect_cluster_metric b/km-dist/init/template/ks_kafka_connect_cluster_metric deleted file mode 100644 index 7fa4c523..00000000 --- a/km-dist/init/template/ks_kafka_connect_cluster_metric +++ /dev/null @@ -1,86 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_connect_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "connectClusterId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "ConnectorCount" : { - "type" : "float" - }, - "TaskCount" : { - "type" : "float" - }, - "ConnectorStartupAttemptsTotal" : { - "type" : "float" - }, - "ConnectorStartupFailurePercentage" : { - "type" : "float" - }, - "ConnectorStartupFailureTotal" : { - "type" : "float" - }, - "ConnectorStartupSuccessPercentage" : { - "type" : "float" - }, - "ConnectorStartupSuccessTotal" : { - "type" : "float" - }, - "TaskStartupAttemptsTotal" : { - "type" : "float" - }, - "TaskStartupFailurePercentage" : { - "type" : "float" - }, - "TaskStartupFailureTotal" : { - "type" : "float" - }, - "TaskStartupSuccessPercentage" : { - "type" : "float" - }, - "TaskStartupSuccessTotal" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_connect_connector_metric b/km-dist/init/template/ks_kafka_connect_connector_metric deleted file mode 100644 index b26836a0..00000000 --- a/km-dist/init/template/ks_kafka_connect_connector_metric +++ /dev/null @@ -1,194 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_connect_connector_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "connectClusterId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "connectorName" : { - "type" : "keyword" - }, - "connectorNameAndClusterId" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "HealthState" : { - "type" : "float" - }, - "ConnectorTotalTaskCount" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ConnectorRunningTaskCount" : { - "type" : "float" - }, - "ConnectorPausedTaskCount" : { - "type" : "float" - }, - "ConnectorFailedTaskCount" : { - "type" : "float" - }, - "ConnectorUnassignedTaskCount" : { - "type" : "float" - }, - "BatchSizeAvg" : { - "type" : "float" - }, - "BatchSizeMax" : { - "type" : "float" - }, - "OffsetCommitAvgTimeMs" : { - "type" : "float" - }, - "OffsetCommitMaxTimeMs" : { - "type" : "float" - }, - "OffsetCommitFailurePercentage" : { - "type" : "float" - }, - "OffsetCommitSuccessPercentage" : { - "type" : "float" - }, - "PollBatchAvgTimeMs" : { - "type" : "float" - }, - "PollBatchMaxTimeMs" : { - "type" : "float" - }, - "SourceRecordActiveCount" : { - "type" : "float" - }, - "SourceRecordActiveCountAvg" : { - "type" : "float" - }, - "SourceRecordActiveCountMax" : { - "type" : "float" - }, - "SourceRecordPollRate" : { - "type" : "float" - }, - "SourceRecordPollTotal" : { - "type" : "float" - }, - "SourceRecordWriteRate" : { - "type" : "float" - }, - "SourceRecordWriteTotal" : { - "type" : "float" - }, - "OffsetCommitCompletionRate" : { - "type" : "float" - }, - "OffsetCommitCompletionTotal" : { - "type" : "float" - }, - "OffsetCommitSkipRate" : { - "type" : "float" - }, - "OffsetCommitSkipTotal" : { - "type" : "float" - }, - "PartitionCount" : { - "type" : "float" - }, - "PutBatchAvgTimeMs" : { - "type" : "float" - }, - "PutBatchMaxTimeMs" : { - "type" : "float" - }, - "SinkRecordActiveCount" : { - "type" : "float" - }, - "SinkRecordActiveCountAvg" : { - "type" : "float" - }, - "SinkRecordActiveCountMax" : { - "type" : "float" - }, - "SinkRecordLagMax" : { - "type" : "float" - }, - "SinkRecordReadRate" : { - "type" : "float" - }, - "SinkRecordReadTotal" : { - "type" : "float" - }, - "SinkRecordSendRate" : { - "type" : "float" - }, - "SinkRecordSendTotal" : { - "type" : "float" - }, - "DeadletterqueueProduceFailures" : { - "type" : "float" - }, - "DeadletterqueueProduceRequests" : { - "type" : "float" - }, - "LastErrorTimestamp" : { - "type" : "float" - }, - "TotalErrorsLogged" : { - "type" : "float" - }, - "TotalRecordErrors" : { - "type" : "float" - }, - "TotalRecordFailures" : { - "type" : "float" - }, - "TotalRecordsSkipped" : { - "type" : "float" - }, - "TotalRetries" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_group_metric b/km-dist/init/template/ks_kafka_group_metric deleted file mode 100644 index 24ff12de..00000000 --- a/km-dist/init/template/ks_kafka_group_metric +++ /dev/null @@ -1,74 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_group_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "6" - } - }, - "mappings" : { - "properties" : { - "group" : { - "type" : "keyword" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "HealthScore" : { - "type" : "float" - }, - "Lag" : { - "type" : "float" - }, - "OffsetConsumed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "groupMetric" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_partition_metric b/km-dist/init/template/ks_kafka_partition_metric deleted file mode 100644 index 51193c22..00000000 --- a/km-dist/init/template/ks_kafka_partition_metric +++ /dev/null @@ -1,65 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "6" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_topic_metric b/km-dist/init/template/ks_kafka_topic_metric deleted file mode 100644 index 4a1aa70c..00000000 --- a/km-dist/init/template/ks_kafka_topic_metric +++ /dev/null @@ -1,116 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_topic_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "6" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "topic" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "BytesIn_min_15" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "BytesRejected" : { - "type" : "float" - }, - "PartitionURP" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ReplicationCount" : { - "type" : "float" - }, - "ReplicationBytesOut" : { - "type" : "float" - }, - "ReplicationBytesIn" : { - "type" : "float" - }, - "FailedFetchRequests" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "LogSize" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "FailedProduceRequests" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "brokerAgg" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_zookeeper_metric b/km-dist/init/template/ks_kafka_zookeeper_metric deleted file mode 100644 index 10c9324b..00000000 --- a/km-dist/init/template/ks_kafka_zookeeper_metric +++ /dev/null @@ -1,84 +0,0 @@ -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_zookeeper_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "AvgRequestLatency" : { - "type" : "double" - }, - "MinRequestLatency" : { - "type" : "double" - }, - "MaxRequestLatency" : { - "type" : "double" - }, - "OutstandingRequests" : { - "type" : "double" - }, - "NodeCount" : { - "type" : "double" - }, - "WatchCount" : { - "type" : "double" - }, - "NumAliveConnections" : { - "type" : "double" - }, - "PacketsReceived" : { - "type" : "double" - }, - "PacketsSent" : { - "type" : "double" - }, - "EphemeralsCount" : { - "type" : "double" - }, - "ApproximateDataSize" : { - "type" : "double" - }, - "OpenFileDescriptorCount" : { - "type" : "double" - }, - "MaxFileDescriptorCount" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - } \ No newline at end of file diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/utils/LoadSQLUtil.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/utils/LoadSQLUtil.java new file mode 100644 index 00000000..50d4e1f3 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/utils/LoadSQLUtil.java @@ -0,0 +1,55 @@ +package com.xiaojukeji.know.streaming.km.persistence.utils; + +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + + +public class LoadSQLUtil { + private static final ILog LOGGER = LogFactory.getLog(LoadSQLUtil.class); + + public static final String SQL_DDL_KS_KM = "sql/ddl-ks-km.sql"; + public static final String SQL_DDL_LOGI_JOB = "sql/ddl-logi-job.sql"; + public static final String SQL_DDL_LOGI_SECURITY = "sql/ddl-logi-security.sql"; + public static final String SQL_DML_KS_KM = "sql/dml-ks-km.sql"; + public static final String SQL_DML_LOGI = "sql/dml-logi.sql"; + + public static String loadSQL(String sqlFileName) { + InputStream inputStream = LoadSQLUtil.class.getClassLoader().getResourceAsStream(sqlFileName); + if (inputStream == null) { + LOGGER.error("method=loadSQL||fileName={}||msg=read script failed", sqlFileName); + return ""; + } + + try { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + String line = null; + + StringBuilder sb = new StringBuilder(); + while ((line = bufferedReader.readLine()) != null) { + sb.append(line).append("\n"); + } + + return sb.toString(); + } catch (Exception e) { + LOGGER.error("method=loadSQL||fileName={}||errMsg={}||msg=read script failed", sqlFileName, e.getMessage()); + } finally { + try { + inputStream.close(); + + + } catch (IOException e) { + LOGGER.error("method=loadSQL||fileName={}||errMsg={}||msg=close reading script failed", sqlFileName, e.getMessage()); + } + } + + return ""; + } + + private LoadSQLUtil() { + } +} diff --git a/km-dist/init/sql/ddl-ks-km.sql b/km-persistence/src/main/resources/sql/ddl-ks-km.sql similarity index 100% rename from km-dist/init/sql/ddl-ks-km.sql rename to km-persistence/src/main/resources/sql/ddl-ks-km.sql diff --git a/km-dist/init/sql/ddl-logi-job.sql b/km-persistence/src/main/resources/sql/ddl-logi-job.sql similarity index 100% rename from km-dist/init/sql/ddl-logi-job.sql rename to km-persistence/src/main/resources/sql/ddl-logi-job.sql diff --git a/km-dist/init/sql/ddl-logi-security.sql b/km-persistence/src/main/resources/sql/ddl-logi-security.sql similarity index 100% rename from km-dist/init/sql/ddl-logi-security.sql rename to km-persistence/src/main/resources/sql/ddl-logi-security.sql diff --git a/km-dist/init/sql/dml-ks-km.sql b/km-persistence/src/main/resources/sql/dml-ks-km.sql similarity index 100% rename from km-dist/init/sql/dml-ks-km.sql rename to km-persistence/src/main/resources/sql/dml-ks-km.sql diff --git a/km-dist/init/sql/dml-logi.sql b/km-persistence/src/main/resources/sql/dml-logi.sql similarity index 100% rename from km-dist/init/sql/dml-logi.sql rename to km-persistence/src/main/resources/sql/dml-logi.sql diff --git a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/KMTestEnvService.java b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/KMTestEnvService.java index acae3123..132fb4db 100644 --- a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/KMTestEnvService.java +++ b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/KMTestEnvService.java @@ -42,8 +42,16 @@ public abstract class KMTestEnvService { @DynamicPropertySource static void setUp(DynamicPropertyRegistry registry) { registry.add("spring.datasource.know-streaming.jdbc-url", mySQLTestContainer.jdbcUrl()); + registry.add("spring.datasource.know-streaming.username", mySQLTestContainer.jdbcUsername()); + registry.add("spring.datasource.know-streaming.password", mySQLTestContainer.jdbcPassword()); + registry.add("spring.logi-job.jdbc-url", mySQLTestContainer.jdbcUrl()); + registry.add("spring.logi-job.username", mySQLTestContainer.jdbcUsername()); + registry.add("spring.logi-job.password", mySQLTestContainer.jdbcPassword()); + registry.add("spring.logi-security.jdbc-url", mySQLTestContainer.jdbcUrl()); + registry.add("spring.logi-security.username", mySQLTestContainer.jdbcUsername()); + registry.add("spring.logi-security.password", mySQLTestContainer.jdbcPassword()); registry.add("es.client.address", esTestContainer.esUrl()); } diff --git a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/es/ESTestContainer.java b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/es/ESTestContainer.java index 2a297252..3ba7e99c 100644 --- a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/es/ESTestContainer.java +++ b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/es/ESTestContainer.java @@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.test.container.es; import com.xiaojukeji.know.streaming.test.container.BaseTestContainer; import org.jetbrains.annotations.NotNull; -import org.testcontainers.containers.GenericContainer; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; @@ -19,14 +18,6 @@ public class ESTestContainer extends BaseTestContainer { .withEnv("ES_JAVA_OPTS", "-Xms512m -Xmx512m") .withEnv("discovery.type", "single-node"); - // km容器，需要初始化es索引模版 - private static final GenericContainer INIT_CONTAINER = new GenericContainer<>( - "knowstreaming/knowstreaming-manager:latest" - ) - .withEnv("TZ", "Asia/Shanghai") - .withCommand("/bin/bash", "/es_template_create.sh") - .dependsOn(ES_CONTAINER); - @NotNull public Supplier esUrl() { return () -> ES_CONTAINER.getHost() + ":" + ES_CONTAINER.getMappedPort(9200); @@ -34,7 +25,7 @@ public class ESTestContainer extends BaseTestContainer { @Override public void init() { - Startables.deepStart(ES_CONTAINER, INIT_CONTAINER).join(); + Startables.deepStart(ES_CONTAINER).join(); } @Override diff --git a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/KSMySQLContainer.java b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/KSMySQLContainer.java index a656e6d9..56a32a7c 100644 --- a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/KSMySQLContainer.java +++ b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/KSMySQLContainer.java @@ -1,11 +1,20 @@ package com.xiaojukeji.know.streaming.test.container.mysql; +import com.didiglobal.logi.log.ILog; +import com.didiglobal.logi.log.LogFactory; +import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import org.jetbrains.annotations.NotNull; import org.testcontainers.containers.ContainerLaunchException; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.delegate.DatabaseDelegate; +import org.testcontainers.ext.ScriptUtils; +import org.testcontainers.jdbc.JdbcDatabaseDelegate; import org.testcontainers.utility.DockerImageName; +import javax.script.ScriptException; +import java.util.ArrayList; +import java.util.List; import java.util.Set; /** @@ -13,6 +22,8 @@ import java.util.Set; * @see org.testcontainers.containers.MySQLContainer */ public class KSMySQLContainer> extends JdbcDatabaseContainer { + private static final ILog LOGGER = LogFactory.getLog(KSMySQLContainer.class); + public static final String NAME = "mysql"; private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("mysql"); @@ -39,6 +50,8 @@ public class KSMySQLContainer> extends JdbcD private static final String MYSQL_ROOT_USER = "root"; + private List> initScriptPathAndContentList = new ArrayList<>(); + /** * @deprecated use {@link MySQLContainer(DockerImageName)} instead */ @@ -169,4 +182,31 @@ public class KSMySQLContainer> extends JdbcD this.password = password; return self(); } + + public SELF addInitScriptPathAndContent(String initScriptPath, String initScriptContent) { + initScriptPathAndContentList.add(new Tuple<>(initScriptPath, initScriptContent)); + return self(); + } + + // KS改动的地方 + @Override + public DatabaseDelegate getDatabaseDelegate() { + return new JdbcDatabaseDelegate(this, ""); + } + + @Override + protected void runInitScriptIfRequired() { + if (initScriptPathAndContentList.isEmpty()) { + return; + } + + for (Tuple elem: initScriptPathAndContentList) { + try { + ScriptUtils.executeDatabaseScript(this.getDatabaseDelegate(), elem.getV1(), elem.getV2()); + } catch (ScriptException var5) { + LOGGER.error("Error while executing init script: {}", elem.getV1(), var5); + throw new ScriptUtils.UncategorizedScriptException("Error while executing init script: " + elem.getV2(), var5); + } + } + } } diff --git a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/MySQLTestContainer.java b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/MySQLTestContainer.java index d232940a..e65e0ec8 100644 --- a/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/MySQLTestContainer.java +++ b/km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/MySQLTestContainer.java @@ -1,5 +1,6 @@ package com.xiaojukeji.know.streaming.test.container.mysql; +import com.xiaojukeji.know.streaming.km.persistence.utils.LoadSQLUtil; import com.xiaojukeji.know.streaming.test.container.BaseTestContainer; import org.jetbrains.annotations.NotNull; import org.testcontainers.lifecycle.Startables; @@ -8,7 +9,12 @@ import org.testcontainers.utility.DockerImageName; import java.util.function.Supplier; public class MySQLTestContainer extends BaseTestContainer { - private static final String DB_PROPERTY = "?useUnicode=true" + + private static final String DB_USERNAME = "root"; + private static final String DB_PASSWORD = "1234567890"; + + private static final String DATABASE_NAME = "know_streaming"; + + private static final String DB_PROPERTY = "?useUnicode=true" + "&characterEncoding=utf8" + "&jdbcCompliantTruncation=true" + "&allowMultiQueries=true" + @@ -18,19 +24,34 @@ public class MySQLTestContainer extends BaseTestContainer { "&allowPublicKeyRetrieval=true"; private static final KSMySQLContainer MYSQL_CONTAINER = new KSMySQLContainer<>( - DockerImageName.parse("knowstreaming/knowstreaming-mysql:latest").asCompatibleSubstituteFor("mysql") + DockerImageName.parse("mysql:5.7").asCompatibleSubstituteFor("mysql") ) - .withEnv("MYSQL_ROOT_HOST", "%") .withEnv("TZ", "Asia/Shanghai") - .withDatabaseName("know_streaming") - .withUsername("root") - .withPassword("mysql_pass"); + .withDatabaseName(DATABASE_NAME) + .withUsername(DB_USERNAME) + .withPassword(DB_PASSWORD) + .addInitScriptPathAndContent(LoadSQLUtil.SQL_DDL_KS_KM, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DDL_KS_KM))) + .addInitScriptPathAndContent(LoadSQLUtil.SQL_DDL_LOGI_JOB, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DDL_LOGI_JOB))) + .addInitScriptPathAndContent(LoadSQLUtil.SQL_DDL_LOGI_SECURITY, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DDL_LOGI_SECURITY))) + .addInitScriptPathAndContent(LoadSQLUtil.SQL_DML_KS_KM, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DML_KS_KM))) + .addInitScriptPathAndContent(LoadSQLUtil.SQL_DML_LOGI, String.format("use %s;\n%s", DATABASE_NAME, LoadSQLUtil.loadSQL(LoadSQLUtil.SQL_DML_LOGI))) + ; + + @NotNull + public Supplier jdbcUsername() { + return () -> DB_USERNAME; + } + + @NotNull + public Supplier jdbcPassword() { + return () -> DB_PASSWORD; + } @NotNull public Supplier jdbcUrl() { return () -> "jdbc:mariadb://" + MYSQL_CONTAINER.getHost() + ":" + MYSQL_CONTAINER.getMappedPort(3306) - + "/know_streaming" + DB_PROPERTY; + + "/" + DATABASE_NAME + DB_PROPERTY; } @Override