Add km module kafka

This commit is contained in:
leewei
2023-02-14 16:27:47 +08:00
parent 229140f067
commit 0b8160a714
4039 changed files with 718112 additions and 46204 deletions

View File

@@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.connect.rest.basic.auth.extension;
import org.apache.kafka.common.utils.AppInfoParser;
import org.apache.kafka.connect.rest.ConnectRestExtension;
import org.apache.kafka.connect.rest.ConnectRestExtensionContext;
import java.io.IOException;
import java.util.Map;
/**
* Provides the ability to authenticate incoming BasicAuth credentials using the configured JAAS {@link
* javax.security.auth.spi.LoginModule}. An entry with the name {@code KafkaConnect} is expected in the JAAS config file configured in the
* JVM. An implementation of {@link javax.security.auth.spi.LoginModule} needs to be provided in the JAAS config file. The {@code
* LoginModule} implementation should configure the {@link javax.security.auth.callback.CallbackHandler} with only {@link
* javax.security.auth.callback.NameCallback} and {@link javax.security.auth.callback.PasswordCallback}.
*
* <p>To use this extension, one needs to add the following config in the {@code worker.properties}
* <pre>
* rest.extension.classes = org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension
* </pre>
*
* <p> An example JAAS config would look as below
* <Pre>
* KafkaConnect {
* org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule required
* file="/mnt/secret/credentials.properties";
* };
*</Pre>
*
* <p>This is a reference implementation of the {@link ConnectRestExtension} interface. It registers an implementation of {@link
* javax.ws.rs.container.ContainerRequestFilter} that does JAAS based authentication of incoming Basic Auth credentials. {@link
* ConnectRestExtension} implementations are loaded via the plugin class loader using {@link java.util.ServiceLoader} mechanism and hence
* the packaged jar includes {@code META-INF/services/org.apache.kafka.connect.rest.extension.ConnectRestExtension} with the entry
* {@code org.apache.kafka.connect.extension.auth.jaas.BasicAuthSecurityRestExtension}
*
* <p><b>NOTE: The implementation ships with a default {@link PropertyFileLoginModule} that helps authenticate the request against a
* property file. {@link PropertyFileLoginModule} is NOT intended to be used in production since the credentials are stored in PLAINTEXT. One can use
* this extension in production by using their own implementation of {@link javax.security.auth.spi.LoginModule} that authenticates against
* stores like LDAP, DB, etc.</b>
*/
public class BasicAuthSecurityRestExtension implements ConnectRestExtension {
@Override
public void register(ConnectRestExtensionContext restPluginContext) {
restPluginContext.configurable().register(JaasBasicAuthFilter.class);
}
@Override
public void close() throws IOException {
}
@Override
public void configure(Map<String, ?> configs) {
}
@Override
public String version() {
return AppInfoParser.getVersion();
}
}

View File

@@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.connect.rest.basic.auth.extension;
import java.util.regex.Pattern;
import javax.ws.rs.HttpMethod;
import org.apache.kafka.common.config.ConfigException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import javax.security.auth.callback.Callback;
import javax.security.auth.callback.CallbackHandler;
import javax.security.auth.callback.NameCallback;
import javax.security.auth.callback.PasswordCallback;
import javax.security.auth.callback.UnsupportedCallbackException;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.core.Response;
public class JaasBasicAuthFilter implements ContainerRequestFilter {
private static final String CONNECT_LOGIN_MODULE = "KafkaConnect";
static final String AUTHORIZATION = "Authorization";
private static final Pattern TASK_REQUEST_PATTERN = Pattern.compile("/?connectors/([^/]+)/tasks/?");
@Override
public void filter(ContainerRequestContext requestContext) throws IOException {
try {
if (!(requestContext.getMethod().equals(HttpMethod.POST) && TASK_REQUEST_PATTERN.matcher(requestContext.getUriInfo().getPath()).matches())) {
LoginContext loginContext =
new LoginContext(CONNECT_LOGIN_MODULE, new BasicAuthCallBackHandler(
requestContext.getHeaderString(AUTHORIZATION)));
loginContext.login();
}
} catch (LoginException | ConfigException e) {
requestContext.abortWith(
Response.status(Response.Status.UNAUTHORIZED)
.entity("User cannot access the resource.")
.build());
}
}
public static class BasicAuthCallBackHandler implements CallbackHandler {
private static final String BASIC = "basic";
private static final char COLON = ':';
private static final char SPACE = ' ';
private String username;
private String password;
public BasicAuthCallBackHandler(String credentials) {
if (credentials != null) {
int space = credentials.indexOf(SPACE);
if (space > 0) {
String method = credentials.substring(0, space);
if (BASIC.equalsIgnoreCase(method)) {
credentials = credentials.substring(space + 1);
credentials = new String(Base64.getDecoder().decode(credentials),
StandardCharsets.UTF_8);
int i = credentials.indexOf(COLON);
if (i > 0) {
username = credentials.substring(0, i);
password = credentials.substring(i + 1);
}
}
}
}
}
@Override
public void handle(Callback[] callbacks) throws UnsupportedCallbackException {
for (Callback callback : callbacks) {
if (callback instanceof NameCallback) {
((NameCallback) callback).setName(username);
} else if (callback instanceof PasswordCallback) {
((PasswordCallback) callback).setPassword(password.toCharArray());
} else {
throw new UnsupportedCallbackException(callback, "Supports only NameCallback "
+ "and PasswordCallback");
}
}
}
}
}

View File

@@ -0,0 +1,119 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.connect.rest.basic.auth.extension;
import org.apache.kafka.common.config.ConfigException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import javax.security.auth.Subject;
import javax.security.auth.callback.Callback;
import javax.security.auth.callback.CallbackHandler;
import javax.security.auth.callback.NameCallback;
import javax.security.auth.callback.PasswordCallback;
import javax.security.auth.login.LoginException;
import javax.security.auth.spi.LoginModule;
/**
* {@link PropertyFileLoginModule} authenticates against a properties file.
* The credentials should be stored in the format {username}={password} in the properties file.
* The absolute path of the file needs to specified using the option <b>file</b>
*
* <p><b>NOTE: This implementation is NOT intended to be used in production since the credentials are stored in PLAINTEXT in the
* properties file.</b>
*/
public class PropertyFileLoginModule implements LoginModule {
private static final Logger log = LoggerFactory.getLogger(PropertyFileLoginModule.class);
private CallbackHandler callbackHandler;
private static final String FILE_OPTIONS = "file";
private String fileName;
private boolean authenticated;
private static Map<String, Properties> credentialPropertiesMap = new ConcurrentHashMap<>();
@Override
public void initialize(Subject subject, CallbackHandler callbackHandler, Map<String, ?> sharedState, Map<String, ?> options) {
this.callbackHandler = callbackHandler;
fileName = (String) options.get(FILE_OPTIONS);
if (fileName == null || fileName.trim().isEmpty()) {
throw new ConfigException("Property Credentials file must be specified");
}
if (!credentialPropertiesMap.containsKey(fileName)) {
Properties credentialProperties = new Properties();
try {
try (InputStream inputStream = Files.newInputStream(Paths.get(fileName))) {
credentialProperties.load(inputStream);
}
credentialPropertiesMap.putIfAbsent(fileName, credentialProperties);
} catch (IOException e) {
log.error("Error loading credentials file ", e);
throw new ConfigException("Error loading Property Credentials file");
}
}
}
@Override
public boolean login() throws LoginException {
Callback[] callbacks = configureCallbacks();
try {
callbackHandler.handle(callbacks);
} catch (Exception e) {
throw new LoginException(e.getMessage());
}
String username = ((NameCallback) callbacks[0]).getName();
char[] passwordChars = ((PasswordCallback) callbacks[1]).getPassword();
String password = passwordChars != null ? new String(passwordChars) : null;
Properties credentialProperties = credentialPropertiesMap.get(fileName);
authenticated = credentialProperties.isEmpty() ||
(password != null && password.equals(credentialProperties.get(username)));
return authenticated;
}
@Override
public boolean commit() throws LoginException {
return authenticated;
}
@Override
public boolean abort() throws LoginException {
return true;
}
@Override
public boolean logout() throws LoginException {
return true;
}
private Callback[] configureCallbacks() {
Callback[] callbacks = new Callback[2];
callbacks[0] = new NameCallback("Enter user name");
callbacks[1] = new PasswordCallback("Enter password", false);
return callbacks;
}
}

View File

@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension

View File

@@ -0,0 +1,200 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.connect.rest.basic.auth.extension;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.UriInfo;
import org.apache.kafka.common.security.JaasUtils;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.easymock.PowerMock;
import org.powermock.api.easymock.annotation.MockStrict;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.modules.junit4.PowerMockRunner;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import javax.security.auth.login.Configuration;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.core.Response;
import static org.powermock.api.easymock.PowerMock.replayAll;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.*")
public class JaasBasicAuthFilterTest {
@MockStrict
private ContainerRequestContext requestContext;
private JaasBasicAuthFilter jaasBasicAuthFilter = new JaasBasicAuthFilter();
private String previousJaasConfig;
private Configuration previousConfiguration;
@MockStrict
private UriInfo uriInfo;
@Before
public void setup() {
EasyMock.reset(requestContext);
}
@After
public void tearDown() {
if (previousJaasConfig != null) {
System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, previousJaasConfig);
}
Configuration.setConfiguration(previousConfiguration);
}
@Test
public void testSuccess() throws IOException {
File credentialFile = File.createTempFile("credential", ".properties");
credentialFile.deleteOnExit();
List<String> lines = new ArrayList<>();
lines.add("user=password");
lines.add("user1=password1");
Files.write(credentialFile.toPath(), lines, StandardCharsets.UTF_8);
setupJaasConfig("KafkaConnect", credentialFile.getPath(), true);
setMock("Basic", "user", "password", false);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testBadCredential() throws IOException {
setMock("Basic", "user1", "password", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testBadPassword() throws IOException {
setMock("Basic", "user", "password1", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testUnknownBearer() throws IOException {
setMock("Unknown", "user", "password", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testUnknownLoginModule() throws IOException {
setupJaasConfig("KafkaConnect1", "/tmp/testcrednetial", true);
Configuration.setConfiguration(null);
setMock("Basic", "user", "password", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testUnknownCredentialsFile() throws IOException {
setupJaasConfig("KafkaConnect", "/tmp/testcrednetial", true);
Configuration.setConfiguration(null);
setMock("Basic", "user", "password", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testEmptyCredentialsFile() throws IOException {
File jaasConfigFile = File.createTempFile("ks-jaas-", ".conf");
jaasConfigFile.deleteOnExit();
System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, jaasConfigFile.getPath());
setupJaasConfig("KafkaConnect", "", true);
Configuration.setConfiguration(null);
setMock("Basic", "user", "password", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testNoFileOption() throws IOException {
File jaasConfigFile = File.createTempFile("ks-jaas-", ".conf");
jaasConfigFile.deleteOnExit();
System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, jaasConfigFile.getPath());
setupJaasConfig("KafkaConnect", "", false);
Configuration.setConfiguration(null);
setMock("Basic", "user", "password", true);
jaasBasicAuthFilter.filter(requestContext);
}
@Test
public void testPostWithoutAppropriateCredential() throws IOException {
EasyMock.expect(requestContext.getMethod()).andReturn(HttpMethod.POST);
EasyMock.expect(requestContext.getUriInfo()).andReturn(uriInfo);
EasyMock.expect(uriInfo.getPath()).andReturn("connectors/connName/tasks");
PowerMock.replayAll();
jaasBasicAuthFilter.filter(requestContext);
EasyMock.verify(requestContext);
}
@Test
public void testPostNotChangingConnectorTask() throws IOException {
EasyMock.expect(requestContext.getMethod()).andReturn(HttpMethod.POST);
EasyMock.expect(requestContext.getUriInfo()).andReturn(uriInfo);
EasyMock.expect(uriInfo.getPath()).andReturn("local:randomport/connectors/connName");
String authHeader = "Basic" + Base64.getEncoder().encodeToString(("user" + ":" + "password").getBytes());
EasyMock.expect(requestContext.getHeaderString(JaasBasicAuthFilter.AUTHORIZATION))
.andReturn(authHeader);
requestContext.abortWith(EasyMock.anyObject(Response.class));
EasyMock.expectLastCall();
PowerMock.replayAll();
jaasBasicAuthFilter.filter(requestContext);
EasyMock.verify(requestContext);
}
private void setMock(String authorization, String username, String password, boolean exceptionCase) {
EasyMock.expect(requestContext.getMethod()).andReturn(HttpMethod.GET);
String authHeader = authorization + " " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
EasyMock.expect(requestContext.getHeaderString(JaasBasicAuthFilter.AUTHORIZATION))
.andReturn(authHeader);
if (exceptionCase) {
requestContext.abortWith(EasyMock.anyObject(Response.class));
EasyMock.expectLastCall();
}
replayAll();
}
private void setupJaasConfig(String loginModule, String credentialFilePath, boolean includeFileOptions) throws IOException {
File jaasConfigFile = File.createTempFile("ks-jaas-", ".conf");
jaasConfigFile.deleteOnExit();
previousJaasConfig = System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, jaasConfigFile.getPath());
List<String> lines;
lines = new ArrayList<>();
lines.add(loginModule + " { org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule required ");
if (includeFileOptions) {
lines.add("file=\"" + credentialFilePath + "\"");
}
lines.add(";};");
Files.write(jaasConfigFile.toPath(), lines, StandardCharsets.UTF_8);
previousConfiguration = Configuration.getConfiguration();
Configuration.setConfiguration(null);
}
}