8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Optimize]优化TestContainers相关依赖(#892)

Browse Source 
1、去除对mysql-connector-j的依赖；
2、整理代码；
This commit is contained in:
zengqiao
2023-02-08 20:40:18 +08:00
committed by EricZeng
parent 915e48de22
commit da95c63503
14 changed files with 375 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
km-rest/src/test/java/com/xiaojukeji/know/streaming/km/KnowStreamApplicationTest.java
View File

@@ -1,7 +1,7 @@
package com.xiaojukeji.know.streaming.km;
import com.xiaojukeji.know.streaming.km.rest.KnowStreaming;
import com.xiaojukeji.know.streaming.test.km.KMBase;
import com.xiaojukeji.know.streaming.test.KMTestEnvService;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.web.server.LocalServerPort;
@@ -19,7 +19,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = KnowStreaming.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class KnowStreamApplicationTest extends KMBase {
public class KnowStreamApplicationTest extends KMTestEnvService {
@LocalServerPort
private Integer port;

3
km-rest/src/test/java/com/xiaojukeji/know/streaming/km/core/service/cluster/ClusterPhyServiceTest.java
View File

@@ -5,6 +5,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterPhyAddDTO
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig;
import com.xiaojukeji.know.streaming.km.common.converter.ClusterConverter;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
import com.xiaojukeji.know.streaming.km.common.exception.DuplicateException;
import com.xiaojukeji.know.streaming.km.common.exception.ParamErrorException;
@@ -33,7 +34,7 @@ public class ClusterPhyServiceTest extends KnowStreamApplicationTest {
ClusterPhyAddDTO dto = new ClusterPhyAddDTO();
dto.setName("test");
dto.setDescription("");
dto.setKafkaVersion(kafkaVersion());
dto.setKafkaVersion(VersionEnum.V_2_5_1.getVersion());
dto.setJmxProperties(jmxConfig);
dto.setClientProperties(properties);
dto.setZookeeper(zookeeperUrl());

74
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/KMTestEnvService.java Normal file
View File

@@ -0,0 +1,74 @@
package com.xiaojukeji.know.streaming.test;
import com.xiaojukeji.know.streaming.test.container.es.ESTestContainer;
import com.xiaojukeji.know.streaming.test.container.kafka.KafkaTestContainer;
import com.xiaojukeji.know.streaming.test.container.mysql.MySQLTestContainer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
public abstract class KMTestEnvService {
private static final boolean useES = true;
private static final boolean useMysql = true;
private static final boolean useKafka = true;
private static MySQLTestContainer mySQLTestContainer;
private static ESTestContainer esTestContainer;
private static KafkaTestContainer kafkaTestContainer;
@BeforeAll
static void init() {
if (useES) {
mySQLTestContainer = new MySQLTestContainer();
mySQLTestContainer.init();
}
if (useMysql) {
esTestContainer = new ESTestContainer();
esTestContainer.init();
}
if (useKafka) {
kafkaTestContainer = new KafkaTestContainer();
kafkaTestContainer.init();
}
}
@DynamicPropertySource
static void setUp(DynamicPropertyRegistry registry) {
registry.add("spring.datasource.know-streaming.jdbc-url", mySQLTestContainer.jdbcUrl());
registry.add("spring.logi-job.jdbc-url", mySQLTestContainer.jdbcUrl());
registry.add("spring.logi-security.jdbc-url", mySQLTestContainer.jdbcUrl());
registry.add("es.client.address", esTestContainer.esUrl());
}
@AfterAll
static void destroy() {
if (mySQLTestContainer != null) {
mySQLTestContainer.cleanup();
}
if (esTestContainer != null) {
esTestContainer.cleanup();
}
if (kafkaTestContainer != null) {
kafkaTestContainer.cleanup();
}
}
protected String bootstrapServers() {
return kafkaTestContainer.getBootstrapServers();
}
protected String zookeeperUrl() {
return kafkaTestContainer.getZKUrl();
}
}

7
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/BaseTestContainer.java Normal file
View File

@@ -0,0 +1,7 @@
package com.xiaojukeji.know.streaming.test.container;
public abstract class BaseTestContainer {
public abstract void init();
public abstract void cleanup();
}

43
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/es/ESTestContainer.java Normal file
View File

@@ -0,0 +1,43 @@
package com.xiaojukeji.know.streaming.test.container.es;
import com.xiaojukeji.know.streaming.test.container.BaseTestContainer;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import java.util.function.Supplier;
public class ESTestContainer extends BaseTestContainer {
// es容器
private static final ElasticsearchContainer ES_CONTAINER = new ElasticsearchContainer(
DockerImageName.parse("docker.io/library/elasticsearch:7.6.2").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch")
)
.withEnv("TZ", "Asia/Shanghai")
.withEnv("ES_JAVA_OPTS", "-Xms512m -Xmx512m")
.withEnv("discovery.type", "single-node");
// km容器，需要初始化es索引模版
private static final GenericContainer<?> INIT_CONTAINER = new GenericContainer<>(
"knowstreaming/knowstreaming-manager:latest"
)
.withEnv("TZ", "Asia/Shanghai")
.withCommand("/bin/bash", "/es_template_create.sh")
.dependsOn(ES_CONTAINER);
@NotNull
public Supplier<Object> esUrl() {
return () -> ES_CONTAINER.getHost() + ":" + ES_CONTAINER.getMappedPort(9200);
}
@Override
public void init() {
Startables.deepStart(ES_CONTAINER, INIT_CONTAINER).join();
}
@Override
public void cleanup() {
}
}

31
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/kafka/KafkaTestContainer.java Normal file
View File

@@ -0,0 +1,31 @@
package com.xiaojukeji.know.streaming.test.container.kafka;
import com.xiaojukeji.know.streaming.test.container.BaseTestContainer;
import org.testcontainers.containers.KafkaContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
public class KafkaTestContainer extends BaseTestContainer {
// kafka容器
private static final KafkaContainer KAFKA_CONTAINER = new KafkaContainer(
DockerImageName.parse("confluentinc/cp-kafka:7.3.1")
).withEnv("TZ", "Asia/Shanghai");
@Override
public void init() {
Startables.deepStart(KAFKA_CONTAINER).join();
}
@Override
public void cleanup() {
}
public String getBootstrapServers() {
return KAFKA_CONTAINER.getBootstrapServers();
}
public String getZKUrl() {
return String.format("%s:%d", KAFKA_CONTAINER.getHost(), KAFKA_CONTAINER.getMappedPort(2181));
}
}

172
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/KSMySQLContainer.java Normal file
View File

@@ -0,0 +1,172 @@
package com.xiaojukeji.know.streaming.test.container.mysql;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.containers.ContainerLaunchException;
import org.testcontainers.containers.JdbcDatabaseContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.utility.DockerImageName;
import java.util.Set;
/**
* @author richardnorth
* @see org.testcontainers.containers.MySQLContainer
*/
public class KSMySQLContainer<SELF extends KSMySQLContainer<SELF>> extends JdbcDatabaseContainer<SELF> {
public static final String NAME = "mysql";
private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("mysql");
@Deprecated
public static final String DEFAULT_TAG = "5.7.34";
@Deprecated
public static final String IMAGE = DEFAULT_IMAGE_NAME.getUnversionedPart();
static final String DEFAULT_USER = "test";
static final String DEFAULT_PASSWORD = "test";
private static final String MY_CNF_CONFIG_OVERRIDE_PARAM_NAME = "TC_MY_CNF";
public static final Integer MYSQL_PORT = 3306;
private String databaseName = "test";
private String username = DEFAULT_USER;
private String password = DEFAULT_PASSWORD;
private static final String MYSQL_ROOT_USER = "root";
/**
* @deprecated use {@link MySQLContainer(DockerImageName)} instead
*/
@Deprecated
public KSMySQLContainer() {
this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG));
}
public KSMySQLContainer(String dockerImageName) {
this(DockerImageName.parse(dockerImageName));
}
public KSMySQLContainer(final DockerImageName dockerImageName) {
super(dockerImageName);
dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME);
addExposedPort(MYSQL_PORT);
}
/**
* @return the ports on which to check if the container is ready
* @deprecated use {@link #getLivenessCheckPortNumbers()} instead
*/
@NotNull
@Override
@Deprecated
protected Set<Integer> getLivenessCheckPorts() {
return super.getLivenessCheckPorts();
}
@Override
protected void configure() {
optionallyMapResourceParameterAsVolume(
MY_CNF_CONFIG_OVERRIDE_PARAM_NAME,
"/etc/mysql/conf.d",
"mysql-default-conf"
);
addEnv("MYSQL_DATABASE", databaseName);
if (!MYSQL_ROOT_USER.equalsIgnoreCase(username)) {
addEnv("MYSQL_USER", username);
}
if (password != null && !password.isEmpty()) {
addEnv("MYSQL_PASSWORD", password);
addEnv("MYSQL_ROOT_PASSWORD", password);
} else if (MYSQL_ROOT_USER.equalsIgnoreCase(username)) {
addEnv("MYSQL_ALLOW_EMPTY_PASSWORD", "yes");
} else {
throw new ContainerLaunchException("Empty password can be used only with the root user");
}
setStartupAttempts(3);
}
@Override
public String getDriverClassName() {
// KS改动的地方
try {
Class.forName("org.mariadb.jdbc.Driver");
return "org.mariadb.jdbc.Driver";
} catch (ClassNotFoundException e) {
return "org.mariadb.jdbc.Driver";
}
}
@Override
public String getJdbcUrl() {
String additionalUrlParams = constructUrlParameters("?", "&");
// KS改动的地方
return "jdbc:mariadb://" + getHost() + ":" + getMappedPort(MYSQL_PORT) + "/" + databaseName + additionalUrlParams;
}
@Override
protected String constructUrlForConnection(String queryString) {
String url = super.constructUrlForConnection(queryString);
if (!url.contains("useSSL=")) {
String separator = url.contains("?") ? "&" : "?";
url = url + separator + "useSSL=false";
}
if (!url.contains("allowPublicKeyRetrieval=")) {
url = url + "&allowPublicKeyRetrieval=true";
}
return url;
}
@Override
public String getDatabaseName() {
return databaseName;
}
@Override
public String getUsername() {
return username;
}
@Override
public String getPassword() {
return password;
}
@Override
public String getTestQueryString() {
return "SELECT 1";
}
public SELF withConfigurationOverride(String s) {
parameters.put(MY_CNF_CONFIG_OVERRIDE_PARAM_NAME, s);
return self();
}
@Override
public SELF withDatabaseName(final String databaseName) {
this.databaseName = databaseName;
return self();
}
@Override
public SELF withUsername(final String username) {
this.username = username;
return self();
}
@Override
public SELF withPassword(final String password) {
this.password = password;
return self();
}
}

44
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/container/mysql/MySQLTestContainer.java Normal file
View File

@@ -0,0 +1,44 @@
package com.xiaojukeji.know.streaming.test.container.mysql;
import com.xiaojukeji.know.streaming.test.container.BaseTestContainer;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import java.util.function.Supplier;
public class MySQLTestContainer extends BaseTestContainer {
private static final String DB_PROPERTY = "?useUnicode=true" +
"&characterEncoding=utf8" +
"&jdbcCompliantTruncation=true" +
"&allowMultiQueries=true" +
"&useSSL=false" +
"&alwaysAutoGeneratedKeys=true" +
"&serverTimezone=GMT%2B8" +
"&allowPublicKeyRetrieval=true";
private static final KSMySQLContainer<?> MYSQL_CONTAINER = new KSMySQLContainer<>(
DockerImageName.parse("knowstreaming/knowstreaming-mysql:latest").asCompatibleSubstituteFor("mysql")
)
.withEnv("MYSQL_ROOT_HOST", "%")
.withEnv("TZ", "Asia/Shanghai")
.withDatabaseName("know_streaming")
.withUsername("root")
.withPassword("mysql_pass");
@NotNull
public Supplier<Object> jdbcUrl() {
return () -> "jdbc:mariadb://"
+ MYSQL_CONTAINER.getHost() + ":" + MYSQL_CONTAINER.getMappedPort(3306)
+ "/know_streaming" + DB_PROPERTY;
}
@Override
public void init() {
Startables.deepStart(MYSQL_CONTAINER).join();
}
@Override
public void cleanup() {
}
}

47
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/kafka/KafkaContainerTest.java
View File

@@ -1,47 +0,0 @@
package com.xiaojukeji.know.streaming.test.kafka;
import com.xiaojukeji.know.streaming.test.kafka.env.KafkaEnv;
import com.xiaojukeji.know.streaming.test.km.env.KMEnv;
import org.testcontainers.containers.KafkaContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
public class KafkaContainerTest implements KafkaEnv {
private static final String KAFKA_VERSION = "7.3.1";
private static final DockerImageName KAFKA_IMAGE = DockerImageName.parse(
"confluentinc/cp-kafka" + KMEnv.SEPARATOR + KAFKA_VERSION);
static KafkaContainer KAFKA_CONTAINER = new KafkaContainer(KAFKA_IMAGE)
.withEnv("TZ", "Asia/Shanghai");
@Override
public void init() {
Startables.deepStart(KAFKA_CONTAINER).join();
}
@Override
public void cleanup() {
/*
* 不需要手动调用清理容器
* 1. test执行结束后testcontainer会清理容器
* 2. junit5的@AfterAll方法会在SpringBoot生命周期结束前执行，导致数据库连接无法关闭
**/
// if (KAFKA_CONTAINER != null) {
// KAFKA_CONTAINER.close();
// }
}
@Override
public String getBootstrapServers() {
return KAFKA_CONTAINER.getBootstrapServers();
}
@Override
public String getZKUrl() {
return String.format("%s:%d", KAFKA_CONTAINER.getHost(), KAFKA_CONTAINER.getMappedPort(2181));
}
@Override
public String getVersion() {
return KAFKA_VERSION;
}
}

13
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/kafka/env/KafkaEnv.java vendored
View File

@@ -1,13 +0,0 @@
package com.xiaojukeji.know.streaming.test.kafka.env;
public interface KafkaEnv {
void init();
void cleanup();
String getBootstrapServers();
String getZKUrl();
String getVersion();
}

69
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/km/KMBase.java
View File

@@ -1,69 +0,0 @@
package com.xiaojukeji.know.streaming.test.km;
import com.xiaojukeji.know.streaming.test.kafka.KafkaContainerTest;
import com.xiaojukeji.know.streaming.test.kafka.env.KafkaEnv;
import com.xiaojukeji.know.streaming.test.km.contrainer.KMContainer;
import com.xiaojukeji.know.streaming.test.km.env.KMEnv;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
public abstract class KMBase {
private static KMEnv kmEnv;
private static KafkaEnv kafkaEnv;
@BeforeAll
static void init() {
if (container()) {
kmEnv = new KMContainer();
kmEnv.init();
if (kmEnv.kafka()) {
kafkaEnv = new KafkaContainerTest();
kafkaEnv.init();
}
}
}
@DynamicPropertySource
static void setUp(DynamicPropertyRegistry registry) {
registry.add("spring.datasource.know-streaming.jdbc-url", KMBase.kmEnv.jdbcUrl());
registry.add("spring.logi-job.jdbc-url", KMBase.kmEnv.jdbcUrl());
registry.add("spring.logi-security.jdbc-url", KMBase.kmEnv.jdbcUrl());
registry.add("spring.logi-security.jdbc-url", KMBase.kmEnv.jdbcUrl());
registry.add("es.client.address", KMBase.kmEnv.esUrl());
}
@AfterAll
static void destroy() {
if (kmEnv != null) {
kmEnv.cleanup();
}
if (kafkaEnv != null) {
kafkaEnv.cleanup();
}
}
static boolean container() {
return true;
}
protected String kafkaVersion() {
return kafkaEnv.getVersion();
}
protected String bootstrapServers() {
return kafkaEnv.getBootstrapServers();
}
protected String zookeeperUrl() {
return kafkaEnv.getZKUrl();
}
}

92
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/km/contrainer/KMContainer.java
View File

@@ -1,92 +0,0 @@
package com.xiaojukeji.know.streaming.test.km.contrainer;
import com.xiaojukeji.know.streaming.test.km.env.KMEnv;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import java.util.function.Supplier;
public class KMContainer implements KMEnv {
private static final String ES_VERSION = "7.6.2";
private static final String LATEST_VERSION = "latest";
private static final String DB_PROPERTY = "?useUnicode=true" +
"&characterEncoding=utf8" +
"&jdbcCompliantTruncation=true" +
"&allowMultiQueries=true" +
"&useSSL=false" +
"&alwaysAutoGeneratedKeys=true" +
"&serverTimezone=GMT%2B8" +
"&allowPublicKeyRetrieval=true";
private static final DockerImageName ES_IMAGE = DockerImageName.parse(
"docker.io/library/elasticsearch" + KMEnv.SEPARATOR + ES_VERSION)
.asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch");
private static final DockerImageName MYSQL_IMAGE = DockerImageName.parse(
"knowstreaming/knowstreaming-mysql" + KMEnv.SEPARATOR + LATEST_VERSION)
.asCompatibleSubstituteFor("mysql");
private static final DockerImageName INIT_IMAGE = DockerImageName.parse(
"knowstreaming/knowstreaming-manager" + KMEnv.SEPARATOR + LATEST_VERSION);
private static final ElasticsearchContainer ES_CONTAINER = new ElasticsearchContainer(ES_IMAGE)
// .withImagePullPolicy(PullPolicy.alwaysPull())
.withEnv("TZ", "Asia/Shanghai")
.withEnv("ES_JAVA_OPTS", "-Xms512m -Xmx512m")
.withEnv("discovery.type", "single-node");
private static final GenericContainer<?> INIT_CONTAINER = new GenericContainer<>(INIT_IMAGE)
.withEnv("TZ", "Asia/Shanghai")
.withCommand("/bin/bash", "/es_template_create.sh")
.dependsOn(ES_CONTAINER);
private static final MySQLContainer<?> MYSQL_CONTAINER = new MySQLContainer<>(MYSQL_IMAGE)
.withEnv("MYSQL_ROOT_HOST", "%")
.withEnv("TZ", "Asia/Shanghai")
.withDatabaseName("know_streaming")
.withUsername("root")
.withPassword("mysql_pass");
@NotNull
public Supplier<Object> jdbcUrl() {
return () -> "jdbc:mariadb://"
+ MYSQL_CONTAINER.getHost() + ":" + MYSQL_CONTAINER.getMappedPort(3306)
+ "/know_streaming" + DB_PROPERTY;
}
@NotNull
public Supplier<Object> esUrl() {
return () -> ES_CONTAINER.getHost() + ":" + ES_CONTAINER.getMappedPort(9200);
}
@Override
public void init() {
if (es()) {
Startables.deepStart(ES_CONTAINER, INIT_CONTAINER).join();
}
if (mysql()) {
Startables.deepStart(MYSQL_CONTAINER).join();
}
}
@Override
public void cleanup() {
/*
* 不需要手动调用清理容器
* 1. test执行结束后testcontainer会清理容器
* 2. junit5的@AfterAll方法会在SpringBoot生命周期结束前执行，导致数据库连接无法关闭
**/
// if (ES_CONTAINER != null) {
// ES_CONTAINER.close();
// }
// if (INIT_CONTAINER != null) {
// INIT_CONTAINER.close();
// }
// if (MYSQL_CONTAINER != null) {
// MYSQL_CONTAINER.close();
// }
}
}

27
km-rest/src/test/java/com/xiaojukeji/know/streaming/test/km/env/KMEnv.java vendored
View File

@@ -1,27 +0,0 @@
package com.xiaojukeji.know.streaming.test.km.env;
import java.util.function.Supplier;
public interface KMEnv {
String SEPARATOR = ":";
void init();
void cleanup();
default boolean es() {
return true;
}
default boolean mysql() {
return true;
}
default boolean kafka() {
return true;
}
Supplier<Object> jdbcUrl();
Supplier<Object> esUrl();
}