Add km module kafka

core/.gitignore

@@ -0,0 +1,3 @@
+.cache-main
+.cache-tests
+/bin/

core/src/main/java/com/didichuxing/datachannel/kafka/config/HAClusterConfig.java

@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.config;
+
+import kafka.api.ApiVersion;
+import kafka.api.ApiVersionValidator$;
+import kafka.server.Defaults;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.config.SecurityConfig;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+
+/**
+ * @author leewei
+ * @date 2021/10/21
+ * The mirror configuration keys
+ */
+public class HAClusterConfig extends AbstractConfig {
+    private static final ConfigDef CONFIG;
+
+    /** <code>didi.kafka.enable</code> */
+    public static final String DIDI_KAFKA_ENABLE_CONFIG = "didi.kafka.enable";
+    private static final String DIDI_KAFKA_ENABLE_DOC = "Enable didi kafka premium features.";
+
+    /** <code>didi.kafka.enable</code> */
+    public static final String DIDI_MIRROR_FETCHER_SPLIT_BATCHES_ENABLE_CONFIG = "didi.mirror.fetcher.split.batches.enable";
+    private static final String DIDI_MIRROR_FETCHER_SPLIT_BATCHES_ENABLE_DOC = "Enable mirror fetcher split multi batch and foreach append to leader.";
+
+    public static final String BROKER_PROTOCOL_VERSION_CONFIG = "broker.protocol.version";
+    public static final String BROKER_PROTOCOL_VERSION_DOC = "Specify which version of the inter-broker protocol will be used.\n" +
+            " This is typically bumped after all brokers were upgraded to a new version.\n" +
+            " Example of some valid values are: 0.8.0, 0.8.1, 0.8.1.1, 0.8.2, 0.8.2.0, 0.8.2.1, 0.9.0.0, 0.9.0.1 Check ApiVersion for the full list.";
+
+    /**
+     * <code>bootstrap.servers</code>
+     */
+    public static final String BOOTSTRAP_SERVERS_CONFIG = CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+
+    /** <code>max.poll.records</code> */
+    public static final String MAX_POLL_RECORDS_CONFIG = "max.poll.records";
+    private static final String MAX_POLL_RECORDS_DOC = "The maximum number of records returned in a single call to poll().";
+
+    /** <code>max.poll.interval.ms</code> */
+    public static final String MAX_POLL_INTERVAL_MS_CONFIG = CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG;
+    private static final String MAX_POLL_INTERVAL_MS_DOC = CommonClientConfigs.MAX_POLL_INTERVAL_MS_DOC;
+    /**
+     * <code>session.timeout.ms</code>
+     */
+    public static final String SESSION_TIMEOUT_MS_CONFIG = CommonClientConfigs.SESSION_TIMEOUT_MS_CONFIG;
+    private static final String SESSION_TIMEOUT_MS_DOC = CommonClientConfigs.SESSION_TIMEOUT_MS_DOC;
+
+    /**
+     * <code>fetch.min.bytes</code>
+     */
+    public static final String FETCH_MIN_BYTES_CONFIG = "fetch.min.bytes";
+    private static final String FETCH_MIN_BYTES_DOC = "The minimum amount of data the server should return for a fetch request. If insufficient data is available the request will wait for that much data to accumulate before answering the request. The default setting of 1 byte means that fetch requests are answered as soon as a single byte of data is available or the fetch request times out waiting for data to arrive. Setting this to something greater than 1 will cause the server to wait for larger amounts of data to accumulate which can improve server throughput a bit at the cost of some additional latency.";
+
+    /**
+     * <code>fetch.max.bytes</code>
+     */
+    public static final String FETCH_MAX_BYTES_CONFIG = "fetch.max.bytes";
+    private static final String FETCH_MAX_BYTES_DOC = "The maximum amount of data the server should return for a fetch request. " +
+            "Records are fetched in batches by the mirror, and if the first record batch in the first non-empty partition of the fetch is larger than " +
+            "this value, the record batch will still be returned to ensure that the mirror can make progress. As such, this is not a absolute maximum. " +
+            "The maximum record batch size accepted by the broker is defined via <code>message.max.bytes</code> (broker config) or " +
+            "<code>max.message.bytes</code> (topic config). Note that the mirror performs multiple fetches in parallel.";
+    public static final int DEFAULT_FETCH_MAX_BYTES = 10 * 1024 * 1024;
+
+    /**
+     * <code>fetch.max.wait.ms</code>
+     */
+    public static final String FETCH_MAX_WAIT_MS_CONFIG = "fetch.max.wait.ms";
+    private static final String FETCH_MAX_WAIT_MS_DOC = "The maximum amount of time the server will block before answering the fetch request if there isn't sufficient data to immediately satisfy the requirement given by fetch.min.bytes.";
+
+    /**
+     * <code>fetch.backoff.ms</code>
+     */
+    public static final String FETCH_BACKOFF_MS_CONFIG = "fetch.backoff.ms";
+    private static final String FETCH_BACKOFF_MS_DOC = "The amount of time to sleep when fetch partition error occurs.";
+
+    /** <code>metadata.max.age.ms</code> */
+    public static final String METADATA_MAX_AGE_CONFIG = CommonClientConfigs.METADATA_MAX_AGE_CONFIG;
+
+    /**
+     * <code>max.partition.fetch.bytes</code>
+     */
+    public static final String MAX_PARTITION_FETCH_BYTES_CONFIG = "max.partition.fetch.bytes";
+    private static final String MAX_PARTITION_FETCH_BYTES_DOC = "The maximum amount of data per-partition the server " +
+            "will return. Records are fetched in batches by the mirror. If the first record batch in the first non-empty " +
+            "partition of the fetch is larger than this limit, the " +
+            "batch will still be returned to ensure that the mirror can make progress. The maximum record batch size " +
+            "accepted by the broker is defined via <code>message.max.bytes</code> (broker config) or " +
+            "<code>max.message.bytes</code> (topic config). See " + FETCH_MAX_BYTES_CONFIG + " for limiting the mirror request size.";
+    public static final int DEFAULT_MAX_PARTITION_FETCH_BYTES = 1 * 1024 * 1024;
+
+    /** <code>send.buffer.bytes</code> */
+    public static final String SEND_BUFFER_CONFIG = CommonClientConfigs.SEND_BUFFER_CONFIG;
+
+    /** <code>receive.buffer.bytes</code> */
+    public static final String RECEIVE_BUFFER_CONFIG = CommonClientConfigs.RECEIVE_BUFFER_CONFIG;
+
+    /**
+     * <code>reconnect.backoff.ms</code>
+     */
+    public static final String RECONNECT_BACKOFF_MS_CONFIG = CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG;
+
+    /**
+     * <code>reconnect.backoff.max.ms</code>
+     */
+    public static final String RECONNECT_BACKOFF_MAX_MS_CONFIG = CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_CONFIG;
+
+    /**
+     * <code>retry.backoff.ms</code>
+     */
+    public static final String RETRY_BACKOFF_MS_CONFIG = CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG;
+
+    /** <code>connections.max.idle.ms</code> */
+    public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG;
+
+    /** <code>request.timeout.ms</code> */
+    public static final String REQUEST_TIMEOUT_MS_CONFIG = CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG;
+    private static final String REQUEST_TIMEOUT_MS_DOC = CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC;
+
+    /** <code>default.api.timeout.ms</code> */
+    public static final String DEFAULT_API_TIMEOUT_MS_CONFIG = CommonClientConfigs.DEFAULT_API_TIMEOUT_MS_CONFIG;
+
+    /**
+     * <code>security.providers</code>
+     */
+    public static final String SECURITY_PROVIDERS_CONFIG = SecurityConfig.SECURITY_PROVIDERS_CONFIG;
+    private static final String SECURITY_PROVIDERS_DOC = SecurityConfig.SECURITY_PROVIDERS_DOC;
+
+    static {
+        CONFIG = new ConfigDef()
+                .define(DIDI_KAFKA_ENABLE_CONFIG,
+                        ConfigDef.Type.BOOLEAN,
+                        false,
+                        ConfigDef.Importance.HIGH,
+                        DIDI_KAFKA_ENABLE_DOC)
+                .define(DIDI_MIRROR_FETCHER_SPLIT_BATCHES_ENABLE_CONFIG,
+                        ConfigDef.Type.BOOLEAN,
+                        false,
+                        ConfigDef.Importance.LOW,
+                        DIDI_MIRROR_FETCHER_SPLIT_BATCHES_ENABLE_DOC)
+                .define(BROKER_PROTOCOL_VERSION_CONFIG,
+                        ConfigDef.Type.STRING,
+                        Defaults.InterBrokerProtocolVersion(),
+                        ApiVersionValidator$.MODULE$,
+                        ConfigDef.Importance.MEDIUM,
+                        BROKER_PROTOCOL_VERSION_DOC)
+                .define(BOOTSTRAP_SERVERS_CONFIG,
+                        ConfigDef.Type.LIST,
+                        Collections.emptyList(),
+                        ConfigDef.Importance.HIGH,
+                        CommonClientConfigs.BOOTSTRAP_SERVERS_DOC)
+                .define(SESSION_TIMEOUT_MS_CONFIG,
+                        ConfigDef.Type.INT,
+                        10000,
+                        ConfigDef.Importance.HIGH,
+                        SESSION_TIMEOUT_MS_DOC)
+                .define(METADATA_MAX_AGE_CONFIG,
+                        ConfigDef.Type.LONG,
+                        5 * 60 * 1000,
+                        atLeast(0),
+                        ConfigDef.Importance.LOW,
+                        CommonClientConfigs.METADATA_MAX_AGE_DOC)
+                .define(MAX_PARTITION_FETCH_BYTES_CONFIG,
+                        ConfigDef.Type.INT,
+                        DEFAULT_MAX_PARTITION_FETCH_BYTES,
+                        atLeast(0),
+                        ConfigDef.Importance.HIGH,
+                        MAX_PARTITION_FETCH_BYTES_DOC)
+                .define(SEND_BUFFER_CONFIG,
+                        ConfigDef.Type.INT,
+                        128 * 1024,
+                        atLeast(CommonClientConfigs.SEND_BUFFER_LOWER_BOUND),
+                        ConfigDef.Importance.MEDIUM,
+                        CommonClientConfigs.SEND_BUFFER_DOC)
+                .define(RECEIVE_BUFFER_CONFIG,
+                        ConfigDef.Type.INT,
+                        64 * 1024,
+                        atLeast(CommonClientConfigs.RECEIVE_BUFFER_LOWER_BOUND),
+                        ConfigDef.Importance.MEDIUM,
+                        CommonClientConfigs.RECEIVE_BUFFER_DOC)
+                .define(FETCH_MIN_BYTES_CONFIG,
+                        ConfigDef.Type.INT,
+                        1,
+                        atLeast(0),
+                        ConfigDef.Importance.HIGH,
+                        FETCH_MIN_BYTES_DOC)
+                .define(FETCH_MAX_BYTES_CONFIG,
+                        ConfigDef.Type.INT,
+                        DEFAULT_FETCH_MAX_BYTES,
+                        atLeast(0),
+                        ConfigDef.Importance.MEDIUM,
+                        FETCH_MAX_BYTES_DOC)
+                .define(FETCH_MAX_WAIT_MS_CONFIG,
+                        ConfigDef.Type.INT,
+                        500,
+                        atLeast(0),
+                        ConfigDef.Importance.LOW,
+                        FETCH_MAX_WAIT_MS_DOC)
+                .define(FETCH_BACKOFF_MS_CONFIG,
+                        ConfigDef.Type.INT,
+                        1000,
+                        atLeast(0),
+                        ConfigDef.Importance.LOW,
+                        FETCH_BACKOFF_MS_DOC)
+                .define(RECONNECT_BACKOFF_MS_CONFIG,
+                        ConfigDef.Type.LONG,
+                        50L,
+                        atLeast(0L),
+                        ConfigDef.Importance.LOW,
+                        CommonClientConfigs.RECONNECT_BACKOFF_MS_DOC)
+                .define(RECONNECT_BACKOFF_MAX_MS_CONFIG,
+                        ConfigDef.Type.LONG,
+                        1000L,
+                        atLeast(0L),
+                        ConfigDef.Importance.LOW,
+                        CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_DOC)
+                .define(RETRY_BACKOFF_MS_CONFIG,
+                        ConfigDef.Type.LONG,
+                        100L,
+                        atLeast(0L),
+                        ConfigDef.Importance.LOW,
+                        CommonClientConfigs.RETRY_BACKOFF_MS_DOC)
+                .define(REQUEST_TIMEOUT_MS_CONFIG,
+                        ConfigDef.Type.INT,
+                        30000,
+                        atLeast(0),
+                        ConfigDef.Importance.MEDIUM,
+                        REQUEST_TIMEOUT_MS_DOC)
+                .define(DEFAULT_API_TIMEOUT_MS_CONFIG,
+                        ConfigDef.Type.INT,
+                        60 * 1000,
+                        atLeast(0),
+                        ConfigDef.Importance.MEDIUM,
+                        CommonClientConfigs.DEFAULT_API_TIMEOUT_MS_DOC)
+                /* default is set to be a bit lower than the server default (10 min), to avoid both client and server closing connection at same time */
+                .define(CONNECTIONS_MAX_IDLE_MS_CONFIG,
+                        ConfigDef.Type.LONG,
+                        9 * 60 * 1000,
+                        ConfigDef.Importance.MEDIUM,
+                        CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_DOC)
+                .define(MAX_POLL_RECORDS_CONFIG,
+                        ConfigDef.Type.INT,
+                        500,
+                        atLeast(1),
+                        ConfigDef.Importance.MEDIUM,
+                        MAX_POLL_RECORDS_DOC)
+                .define(MAX_POLL_INTERVAL_MS_CONFIG,
+                        ConfigDef.Type.INT,
+                        300000,
+                        atLeast(1),
+                        ConfigDef.Importance.MEDIUM,
+                        MAX_POLL_INTERVAL_MS_DOC)
+                // security support
+                .define(SECURITY_PROVIDERS_CONFIG,
+                        ConfigDef.Type.STRING,
+                        null,
+                        ConfigDef.Importance.LOW,
+                        SECURITY_PROVIDERS_DOC)
+                .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+                        ConfigDef.Type.STRING,
+                        CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+                        ConfigDef.Importance.MEDIUM,
+                        CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+                .withClientSslSupport()
+                .withClientSaslSupport();
+    }
+
+    @Override
+    protected Map<String, Object> postProcessParsedConfig(final Map<String, Object> parsedValues) {
+        return CommonClientConfigs.postProcessReconnectBackoffConfigs(this, parsedValues);
+    }
+
+    public HAClusterConfig(Properties props) {
+        super(CONFIG, props);
+    }
+
+    public HAClusterConfig(Map<String, Object> props) {
+        super(CONFIG, props);
+    }
+
+    protected HAClusterConfig(Map<?, ?> props, boolean doLog) {
+        super(CONFIG, props, doLog);
+    }
+
+    public Boolean isDidiKafka() {
+        return getBoolean(DIDI_KAFKA_ENABLE_CONFIG);
+    }
+
+    public Boolean isSplitFetchedBatches() {
+        return getBoolean(DIDI_MIRROR_FETCHER_SPLIT_BATCHES_ENABLE_CONFIG);
+    }
+
+    public ApiVersion brokerProtocolVersion() {
+        String brokerProtocolVersionString = getString(BROKER_PROTOCOL_VERSION_CONFIG);
+        return ApiVersion.apply(brokerProtocolVersionString);
+    }
+
+    public static Set<String> configNames() {
+        return CONFIG.names();
+    }
+
+    public static ConfigDef configDef() {
+        return  new ConfigDef(CONFIG);
+    }
+
+    public static void main(String[] args) {
+        System.out.println(CONFIG.toHtml());
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/HAGroupConfig.java

@@ -0,0 +1,62 @@
+package com.didichuxing.datachannel.kafka.config;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * @author 杨阳
+ * @date {2022/6/1}
+ */
+public class HAGroupConfig extends AbstractConfig {
+    private static final ConfigDef CONFIG;
+
+    /** <code>didi.kafka.enable</code> */
+    public static final String DIDI_KAFKA_ENABLE_CONFIG = "didi.kafka.enable";
+    private static final String DIDI_KAFKA_ENABLE_DOC = "Enable didi kafka premium features.";
+
+    /** <code>didi.ha.remote.cluster</code> */
+    public static final String DIDI_HA_REMOTE_CLUSTER_CONFIG = "didi.ha.remote.cluster";
+    public static final String DIDI_HA_REMOTE_CLUSTER_DOC = "This configuration controls the cluster in which the group to be synchronized exists.";
+
+    static {
+        CONFIG = new ConfigDef()
+                .define(DIDI_KAFKA_ENABLE_CONFIG,
+                        ConfigDef.Type.BOOLEAN,
+                        false,
+                        ConfigDef.Importance.HIGH,
+                        DIDI_KAFKA_ENABLE_DOC)
+                .define(DIDI_HA_REMOTE_CLUSTER_CONFIG,
+                        ConfigDef.Type.STRING,
+                        null,
+                        ConfigDef.Importance.MEDIUM,
+                        DIDI_HA_REMOTE_CLUSTER_DOC);
+    }
+
+    public HAGroupConfig(Properties props) {
+        super(CONFIG, props);
+    }
+
+    public HAGroupConfig(Map<String, Object> props) {
+        super(CONFIG, props);
+    }
+
+    protected HAGroupConfig(Map<?, ?> props, boolean doLog) {
+        super(CONFIG, props, doLog);
+    }
+
+    public Boolean isDidiKafka() {
+        return getBoolean(DIDI_KAFKA_ENABLE_CONFIG);
+    }
+
+    public static Set<String> configNames() {
+        return CONFIG.names();
+    }
+
+    public static ConfigDef configDef() {
+        return new ConfigDef(CONFIG);
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/HATopicConfig.java

@@ -0,0 +1,80 @@
+package com.didichuxing.datachannel.kafka.config;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * @author 杨阳
+ * @date {2022/6/7}
+ */
+public class HATopicConfig extends AbstractConfig {
+    private static final ConfigDef CONFIG;
+
+    public static final String DIDI_HA_REMOTE_CLUSTER = "didi.ha.remote.cluster";
+    public static final String DIDI_HA_REMOTE_CLUSTER_DOC = "This configuration controls the cluster in which the topic to be synchronized exists.";
+
+    public static final String DIDI_HA_REMOTE_TOPIC = "didi.ha.remote.topic";
+    public static final String DIDI_HA_REMOTE_TOPIC_DOC = "This configuration controls the topic to be synchronized.";
+
+    public static final String DIDI_HA_SYNC_TOPIC_PARTITIONS_ENABLED = "didi.ha.sync.topic.partitions.enabled";
+    public static final String DIDI_HA_SYNC_TOPIC_PARTITIONS_ENABLED_DOC = "This configuration controls whether the topic partitions are to be synchronized.";
+
+    public static final String DIDI_HA_SYNC_TOPIC_CONFIGS_ENABLED = "didi.ha.sync.topic.configs.enabled";
+    public static final String DIDI_HA_SYNC_TOPIC_CONFIGS_ENABLED_DOC = "This configuration controls whether the topic configs are to be synchronized.";
+
+    public static final String DIDI_HA_SYNC_TOPIC_ACLS_ENABLED = "didi.ha.sync.topic.acls.enabled";
+    public static final String DIDI_HA_SYNC_TOPIC_ACLS_ENABLED_DOC = "This configuration controls whether the topic acls are to be synchronized.";
+
+    static {
+        CONFIG = new ConfigDef()
+                .define(DIDI_HA_REMOTE_CLUSTER,
+                        ConfigDef.Type.STRING,
+                        null,
+                        ConfigDef.Importance.LOW,
+                        DIDI_HA_REMOTE_CLUSTER_DOC)
+                .define(DIDI_HA_REMOTE_TOPIC,
+                        ConfigDef.Type.STRING,
+                        null,
+                        ConfigDef.Importance.LOW,
+                        DIDI_HA_REMOTE_TOPIC_DOC)
+                .define(DIDI_HA_SYNC_TOPIC_PARTITIONS_ENABLED,
+                        ConfigDef.Type.BOOLEAN,
+                        false,
+                        ConfigDef.Importance.LOW,
+                        DIDI_HA_SYNC_TOPIC_PARTITIONS_ENABLED_DOC)
+                .define(DIDI_HA_SYNC_TOPIC_CONFIGS_ENABLED,
+                        ConfigDef.Type.BOOLEAN,
+                        false,
+                        ConfigDef.Importance.LOW,
+                        DIDI_HA_SYNC_TOPIC_CONFIGS_ENABLED_DOC)
+                .define(DIDI_HA_SYNC_TOPIC_ACLS_ENABLED,
+                        ConfigDef.Type.BOOLEAN,
+                        false,
+                        ConfigDef.Importance.LOW,
+                        DIDI_HA_SYNC_TOPIC_ACLS_ENABLED_DOC);
+    }
+
+    public HATopicConfig(Properties props) {
+        super(CONFIG, props);
+    }
+
+    public HATopicConfig(Map<String, Object> props) {
+        super(CONFIG, props);
+    }
+
+    protected HATopicConfig(Map<?, ?> props, boolean doLog) {
+        super(CONFIG, props, doLog);
+    }
+
+    public static Set<String> configNames() {
+        return CONFIG.names();
+    }
+
+    public static ConfigDef configDef() {
+        return new ConfigDef(CONFIG);
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/HAUserConfig.java

@@ -0,0 +1,50 @@
+package com.didichuxing.datachannel.kafka.config;
+
+import org.apache.kafka.common.config.AbstractConfig;
+import org.apache.kafka.common.config.ConfigDef;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * @author leewei
+ * @date 2022/6/28
+ */
+public class HAUserConfig extends AbstractConfig {
+    public static final String DIDI_HA_ACTIVE_CLUSTER_CONFIG = "didi.ha.active.cluster";
+    public static final String DIDI_HA_ACTIVE_CLUSTER_DOC = "A cluster to be bound by user's AuthId.";
+
+    private static final ConfigDef CONFIG;
+
+    static {
+        CONFIG = new ConfigDef()
+                .define(DIDI_HA_ACTIVE_CLUSTER_CONFIG,
+                        ConfigDef.Type.STRING,
+                        null,
+                        ConfigDef.Importance.HIGH,
+                        DIDI_HA_ACTIVE_CLUSTER_DOC);
+    }
+
+    public HAUserConfig(Properties props) {
+        super(CONFIG, props);
+    }
+
+    public HAUserConfig(Map<String, Object> props) {
+        super(CONFIG, props);
+    }
+
+    protected HAUserConfig(Map<?, ?> props, boolean doLog) {
+        super(CONFIG, props, doLog);
+    }
+
+    public static Set<String> configNames() {
+        return CONFIG.names();
+    }
+
+    public static ConfigDef configDef() {
+        return new ConfigDef(CONFIG);
+    }
+
+
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/MirrorSyncClient.java

@@ -0,0 +1,546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.config;
+
+import com.didichuxing.datachannel.kafka.config.manager.ClusterConfigManager;
+import kafka.server.MirrorTopic;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.Config;
+import org.apache.kafka.clients.admin.ConfigEntry;
+import org.apache.kafka.clients.admin.DescribeConfigsOptions;
+import org.apache.kafka.common.acl.AccessControlEntryFilter;
+import org.apache.kafka.common.acl.AclBinding;
+import org.apache.kafka.common.acl.AclBindingFilter;
+import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.resource.PatternType;
+import org.apache.kafka.common.resource.ResourcePattern;
+import org.apache.kafka.common.resource.ResourcePatternFilter;
+import org.apache.kafka.common.resource.ResourceType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+
+/**
+ * Created by IntelliJ IDEA.
+ * User: didi
+ * Date: 2022/1/12
+ */
+public class MirrorSyncClient implements Closeable {
+
+    private HashMap<String, List<AclBinding>> cachedAcls = new HashMap<>();
+
+    private HashMap<String, Properties> loadedClusterProperties = new HashMap<>();
+    private HashMap<String, AdminClient> adminClientMap = new HashMap<>();
+
+    private String targetClusterId;
+
+    private static final Logger log = LoggerFactory.getLogger(MirrorSyncClient.class);
+    private static final ResourcePatternFilter ANY_TOPIC = new ResourcePatternFilter(ResourceType.TOPIC,
+            null, PatternType.ANY);
+    private static final AclBindingFilter ANY_TOPIC_ACL = new AclBindingFilter(ANY_TOPIC, AccessControlEntryFilter.ANY);
+
+    private static final String DiDiCustomTopicConfigPropertiesPrefix = "didi.ha";
+
+    public MirrorSyncClient(Set<String> clusterSet, String targetClusterId) {
+        this.targetClusterId = targetClusterId;
+        HashSet<String> allSet = new HashSet<>();
+        allSet.addAll(clusterSet);
+        allSet.add(targetClusterId);
+        initAllAdminClient(allSet);
+    }
+
+    /**
+     * init all cluster AdminClient also include targetClusterId
+     * @param clusterSet
+     */
+    protected void initAllAdminClient(Set<String> clusterSet) {
+        clusterSet.forEach(clusterId -> {
+            Properties properties = ClusterConfigManager.getConfigs(clusterId);
+            if (!properties.isEmpty()) {
+                createAdminClient(clusterId, properties);
+            }
+        });
+    }
+
+    /**
+     * release all remote cluster AdminClient
+     */
+    protected void releaseAllRemoteAdminClient() {
+        adminClientMap.entrySet().stream().filter(x -> !x.getKey().equals(targetClusterId)).forEach(y -> {
+            releaseAdminClient(y.getKey(), y.getValue());
+        });
+    }
+
+    /**
+     * release  remote cluster adminClient by clusterId Set
+     *
+     * @param clusterSet
+     */
+    protected void releaseRemovedRemoteAdminClient(Set<String> clusterSet) {
+        clusterSet.forEach(x -> {
+           releaseAdminClient(x, adminClientMap.get(x));
+        });
+    }
+
+    /**
+     * prepare Remote AdminClient when start sync acls and sync configs
+     *
+     * @param clusterSet
+     */
+    public void prepareRemoteAdminClient(Set<String> clusterSet) {
+        if (clusterSet.isEmpty()) {
+            releaseAllRemoteAdminClient();
+        } else {
+            // check 不再需要维持的 clusterId 的AdminClient
+            Set<String> removedSet = adminClientMap.keySet().stream()
+                    .filter(x -> !x.equals(targetClusterId))
+                    .filter(y -> !clusterSet.contains(y)).collect(Collectors.toSet());
+            if (!removedSet.isEmpty()) {
+                releaseRemovedRemoteAdminClient(removedSet);
+            }
+            reloadClusterConfig(clusterSet);
+        }
+    }
+
+    /**
+     *  reload one clusterId AdminClient
+     *  case  clusterid in mirrorTopic but cluster config is removed need release AdminClinet
+     *  case  clusterid is not exists in clientMap && clusterid config is exist need to create
+     *  csae  clusterid is exists in client && clusterid config is updated we need release  and create
+     *
+     * @param clusterId
+     * @return
+     */
+    private boolean reloadAdminClient(String clusterId) {
+        AdminClient client = adminClientMap.get(clusterId);
+        Properties newProperties = ClusterConfigManager.getConfigs(clusterId);
+        if (client == null) {
+            if (!newProperties.isEmpty()) {
+                createAdminClient(clusterId, newProperties);
+            }
+        } else {
+            if (newProperties.isEmpty()) {
+                log.debug("cluster {} will released admin client", clusterId);
+                releaseAdminClient(clusterId, client);
+            } else {
+                if (loadedClusterProperties.get(clusterId) != null && !newProperties.equals(loadedClusterProperties.get(clusterId))) {
+                    log.info("cluster {} will reload admin client use new properties {}", clusterId, newProperties);
+                    releaseAdminClient(clusterId, client);
+                    createAdminClient(clusterId, newProperties);
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * reload all AdminClient in clusterIds
+     * it always used when prepare AdminClient or clusterid config is change notify to reload
+     * @param clusterids
+     */
+    public synchronized void reloadClusterConfig(Set<String> clusterids) {
+        // reload cluster by id
+        clusterids.forEach(this::reloadAdminClient);
+    }
+
+    /**
+     * validate adminclient if created return true else return false
+     *
+     * @param clusterId
+     * @return
+     */
+    protected boolean validateClusterId(String clusterId) {
+        return adminClientMap.containsKey(clusterId);
+    }
+
+    /**
+     * sync acls
+     *
+     * sync topic acls from source to target
+     *
+     * delete unknows topic acls and add exists acls
+     *
+     * @param topics
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    public synchronized void syncTopicAcls(List<MirrorTopic> topics) {
+        try {
+            Map<String, List<MirrorTopic>> grouped = topics.stream().collect(Collectors.groupingBy(MirrorTopic::remoteCluster));
+            grouped.forEach((key, value) -> {
+                try {
+                    if (validateClusterId(key) && validateClusterId(targetClusterId)) {
+                        syncClusterTopicAcls(key, value.stream().collect(Collectors.toMap(MirrorTopic::remoteTopic, x -> x)));
+                    } else {
+                        if (!validateClusterId(key)) {
+                            log.warn("sync config from source cluster {} adminClient is not initialed. maybe cluster config is not set or invalidated", key);
+                        }
+                        if (!validateClusterId(targetClusterId)) {
+                            log.warn("sync config to target cluster {} adminClient is not initialed. maybe cluster config is not set or invalidated", targetClusterId);
+                        }
+                    }
+                } catch (Exception e) {
+                    log.error("sync topic {} ", value, e);
+                }
+            });
+        } catch (Exception e) {
+            log.error("sync topic alcs {} ", topics, e);
+        }
+    }
+
+    /**
+     * sync topics config from source to target
+     *
+     * don't sync high version to low version maybe invalidated
+     *
+     * @param topics
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    public synchronized void syncTopicConfigs(List<MirrorTopic> topics) {
+        try {
+            Map<String, List<MirrorTopic>> grouped = topics.stream().collect(Collectors.groupingBy(MirrorTopic::remoteCluster));
+            grouped.forEach((key, value) -> {
+                try {
+                    if (validateClusterId(key) && validateClusterId(targetClusterId)) {
+                        updateClusterTopicConfigsSimple(key, value.stream().collect(Collectors.toMap(MirrorTopic::remoteTopic, x -> x)));
+                    } else {
+                        if (!validateClusterId(key)) {
+                            log.warn("sync config from source cluster {} adminClient is not initialed. maybe cluster config is not set or invalidated", key);
+                        }
+                        if (!validateClusterId(targetClusterId)) {
+                            log.warn("sync config to target cluster {} adminClient is not initialed. maybe cluster config is not set or invalidated", targetClusterId);
+                        }
+                    }
+                } catch (Exception e) {
+                    log.error("sync topic config failure error {}", value, e);
+                }
+            });
+        } catch (Exception e) {
+            log.error("sync topic config {} ", topics, e);
+        }
+    }
+
+    /**
+     *
+     * 同步集群 cluster 对应 topics 的acls
+     *
+     * @param clusterId
+     * @param topics
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private void syncClusterTopicAcls(String clusterId, Map<String, MirrorTopic> topics) throws InterruptedException, ExecutionException {
+        Collection<AclBinding> result = listTopicAclBindings(clusterId);
+        List<AclBinding> bindings = result.stream()
+                .filter(x -> x.pattern().resourceType() == ResourceType.TOPIC)
+                .filter(x -> x.pattern().patternType() == PatternType.LITERAL)
+                .filter(x -> topics.keySet().contains(x.pattern().name()))
+                .map(x -> new AclBinding(new ResourcePattern(x.pattern().resourceType(), topics.get(x.pattern().name()).localTopic(), x.pattern().patternType()), x.entry()))
+                .collect(Collectors.toList());
+
+        log.debug("sync acls from cluster {} for topics {}", clusterId, topics);
+        List<AclBinding> postedAcls = new ArrayList<>();
+        if (cachedAcls.get(clusterId) != null) {
+            postedAcls.addAll(this.cachedAcls.get(clusterId));
+            postedAcls.removeAll(bindings);
+            if (!postedAcls.isEmpty()) {
+                deleteTopicAcls(postedAcls);
+            }
+        }
+        if (!bindings.isEmpty()) {
+            if (!bindings.equals(this.cachedAcls.get(clusterId))) {
+                updateClusterTopicAcls(bindings);
+            }
+        }
+        cachedAcls.put(clusterId, bindings);
+    }
+
+
+    /**
+     * 更新集群ACL
+     *
+     * @param bindings
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private void updateClusterTopicAcls(List<AclBinding> bindings)
+            throws InterruptedException, ExecutionException {
+        AdminClient client = adminClientMap.get(targetClusterId);
+        if (client != null) {
+            client.createAcls(bindings).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+                if (e != null) {
+                    log.warn("Could not sync ACL of topic {}.", k.pattern().name(), e);
+                } else {
+                    log.info("sync ACL of topic {} success", k.pattern().name());
+                }
+            }));
+        }
+    }
+
+    /**
+     * 获取 集群的 topic config
+     *
+     * @param clusterId
+     * @param topics
+     * @return
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private Map<String, Config> describeTopicConfigs(String clusterId, Map<String, MirrorTopic> topics)
+            throws InterruptedException, ExecutionException {
+        AdminClient client = adminClientMap.get(clusterId);
+        if (client != null) {
+            Set<ConfigResource> resources = topics.values().stream().filter(topic -> !topic.remoteTopic().equals(""))
+                    .map(x -> new ConfigResource(ConfigResource.Type.TOPIC, x.remoteTopic()))
+                    .collect(Collectors.toSet());
+            DescribeConfigsOptions describeOptions = new DescribeConfigsOptions().includeSynonyms(true);
+            return client.describeConfigs(resources, describeOptions).all().get().entrySet().stream()
+                    .collect(Collectors.toMap(x ->  topics.get(x.getKey().name()).localTopic(), x -> x.getValue()));
+        } else {
+            log.warn("can not list topic configs from cluster {}", clusterId);
+            return new HashMap<String, Config>();
+        }
+    }
+
+    /**
+     * describe local cluster topic configs
+     *
+     * @param topics
+     * @return
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private Map<String, Config> describeLocalConfigs(Map<String, MirrorTopic> topics) throws InterruptedException, ExecutionException {
+        AdminClient client = adminClientMap.get(targetClusterId);
+        if (client != null) {
+            Set<ConfigResource> resources = topics.values().stream()
+                    .map(x -> new ConfigResource(ConfigResource.Type.TOPIC, x.localTopic()))
+                    .collect(Collectors.toSet());
+            DescribeConfigsOptions describeOptions = new DescribeConfigsOptions().includeSynonyms(true);
+            return client.describeConfigs(resources, describeOptions).all().get().entrySet().stream()
+                    .collect(Collectors.toMap(x -> x.getKey().name(), x -> x.getValue()));
+        } else {
+            return new HashMap<String, Config>();
+        }
+    }
+
+    /**
+     * 简单模式，直接merge source and target config properties map
+     *
+     * @param clusterId
+     * @param topics
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private void updateClusterTopicConfigsSimple(String clusterId, Map<String, MirrorTopic> topics) throws InterruptedException, ExecutionException {
+        log.debug("sync from cluster {} for topics {}", clusterId, topics);
+        final Map<String, Config> sourceConfigs = describeTopicConfigs(clusterId, topics);
+        final Map<String, Config> localConfigs = describeLocalConfigs(topics);
+        final boolean[] needUpdate = {false};
+        HashMap<String, ArrayList<ConfigEntry>> updateConfigMap = new HashMap<String, ArrayList<ConfigEntry>>();
+        localConfigs.entrySet().stream().forEach(entry -> {
+            Config config = entry.getValue();
+            if (sourceConfigs.containsKey(entry.getKey())) {
+                Config sourceConfig = sourceConfigs.get(entry.getKey());
+                ArrayList<ConfigEntry> merged = new ArrayList<ConfigEntry>();
+                HashMap<String, ConfigEntry> list = sourceConfig.entries().stream()
+                        .filter(item -> !item.name().startsWith(DiDiCustomTopicConfigPropertiesPrefix))
+                        .filter(property -> property.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG)
+                        .collect(Collectors.toMap(item -> item.name(), item -> item, (n1, n2) -> n1, HashMap::new));
+                //add local not exist config entry
+                list.entrySet().stream().forEach(x -> {
+                    if (!config.entries().stream().filter(property -> property.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG).map(y -> y.name()).collect(Collectors.toSet()).contains(x.getKey())) {
+                        needUpdate[0] = true;
+                        merged.add(x.getValue());
+                    }
+                });
+                //update local exist config
+                config.entries()
+                        .stream()
+                        .filter(property -> property.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG)
+                        .forEach(z -> {
+                            if (list.keySet().contains(z.name())) {
+                                merged.add(list.get(z.name()));
+                                // update value from remote
+                                if (!list.get(z.name()).value().equals(z.value())) {
+                                    needUpdate[0] = true;
+                                }
+                            } else {
+                                if (z.name().startsWith(DiDiCustomTopicConfigPropertiesPrefix)) {
+                                    merged.add(z);
+                                } else {
+                                    // local exist but remote removed
+                                    needUpdate[0] = true;
+                                }
+                            }
+                        });
+                if (needUpdate[0]) {
+                    updateConfigMap.put(entry.getKey(), merged);
+                    needUpdate[0] = false;
+                }
+            }
+        });
+
+        if (!updateConfigMap.isEmpty()) {
+            log.info("remote config changed. update config cluster {} for topics {}", clusterId, topics);
+            updateTopicConfigs(convertResourceMap(updateConfigMap));
+        }
+    }
+
+    private Map<String, Config> convertResourceMap(HashMap<String, ArrayList<ConfigEntry>> topicConfigMap) {
+        HashMap<String, Config> maps = new HashMap<String, Config>();
+        topicConfigMap.entrySet().stream().forEach(item -> {
+            maps.put(item.getKey(), new Config(item.getValue()));
+        });
+        return maps;
+    }
+
+    /**
+     * list cluster all acls
+     *
+     * @param clusterId
+     * @return
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private Collection<AclBinding> listTopicAclBindings(String clusterId)
+            throws InterruptedException, ExecutionException {
+        AdminClient client = adminClientMap.get(clusterId);
+        if (client != null) {
+            return client.describeAcls(ANY_TOPIC_ACL).values().get();
+        }
+        return Collections.emptySet();
+    }
+
+
+    /**
+     * 更新 config
+     *
+     * @param topicConfigs
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    // use deprecated alterConfigs API for broker compatibility back to 0.11.0
+    private void updateTopicConfigs(Map<String, Config> topicConfigs)
+            throws InterruptedException, ExecutionException {
+        AdminClient client = adminClientMap.get(targetClusterId);
+        if (client != null) {
+            Map<ConfigResource, Config> configs = topicConfigs.entrySet().stream()
+                    .collect(Collectors.toMap(x ->
+                            new ConfigResource(ConfigResource.Type.TOPIC, x.getKey()), x -> x.getValue()));
+
+            client.alterConfigs(configs).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+                if (e != null) {
+                    log.warn("Could not alter configuration of topic {} ", k.name(), e);
+                } else {
+                    log.debug("sync topic {} config success", k.name());
+                }
+            }));
+        }
+    }
+
+    /**
+     * delete topic acls
+     *
+     * @param bindings
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    private void deleteTopicAcls(List<AclBinding> bindings) throws InterruptedException, ExecutionException {
+        AdminClient client = adminClientMap.get(targetClusterId);
+        if (client != null) {
+            Collection<AclBindingFilter> filters = bindings.stream().map(item -> item.toFilter()).collect(Collectors.toList());
+            client.deleteAcls(filters).values().forEach((k, v) -> v.whenComplete((x, e) -> {
+                if (e != null) {
+                    log.warn("Could not delete ACL of topic {} ", k.patternFilter().name(), e);
+                } else {
+                    log.debug("delete topic {} acls successful",k.patternFilter().name());
+                }
+            }));
+        }
+    }
+
+    /**
+     * create cluster adminClient
+     *
+     * @param sourceClusterId
+     * @param properties  cluster config
+     * @return
+     */
+    private boolean createAdminClient(String sourceClusterId, Properties properties) {
+        AdminClient client = null;
+        try {
+            client = AdminClient.create(properties);
+            if (client != null) {
+                loadedClusterProperties.put(sourceClusterId, properties);
+                adminClientMap.put(sourceClusterId, client);
+                log.debug("create cluster {} admin client properties use {}", sourceClusterId, properties);
+                return true;
+            }
+        } catch (Exception ex) {
+            log.error("can not init source adminclient for cluster {} with {} ", sourceClusterId, properties, ex);
+        }
+        return false;
+    }
+
+    /**
+     * release cluster AdminClient
+     *
+     * @param client
+     */
+    private void releaseAdminClient(String clusterId, AdminClient client) {
+        try {
+            if (client != null) {
+                client.close();
+            }
+        } catch (Exception e) {
+            log.error("release client failure ", e);
+        } finally {
+            adminClientMap.remove(clusterId);
+            loadedClusterProperties.remove(clusterId);
+            cachedAcls.remove(clusterId);
+        }
+    }
+
+    /**
+     * Closes this stream and releases any system resources associated
+     * with it. If the stream is already closed then invoking this
+     * method has no effect.
+     *
+     * <p> As noted in {@link AutoCloseable#close()}, cases where the
+     * close may fail require careful attention. It is strongly advised
+     * to relinquish the underlying resources and to internally
+     * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+     * the {@code IOException}.
+     *
+     * @throws IOException if an I/O error occurs
+     */
+    @Override
+    public void close() throws IOException {
+        // don't use adminClientMap.forEach
+        adminClientMap.entrySet().stream().forEach(y -> {
+            releaseAdminClient(y.getKey(), y.getValue());
+        });
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/MirrorSyncConfig.java

@@ -0,0 +1,215 @@
+package com.didichuxing.datachannel.kafka.config;
+
+import com.didichuxing.datachannel.kafka.config.manager.ClusterConfigManager;
+import kafka.server.AdminManager;
+import kafka.server.MirrorTopic;
+import org.apache.kafka.clients.admin.*;
+import org.apache.kafka.common.acl.AccessControlEntryFilter;
+import org.apache.kafka.common.acl.AclBinding;
+import org.apache.kafka.common.acl.AclBindingFilter;
+import org.apache.kafka.common.config.ConfigResource;
+import org.apache.kafka.common.requests.DescribeConfigsResponse;
+import org.apache.kafka.common.resource.PatternType;
+import org.apache.kafka.common.resource.ResourcePattern;
+import org.apache.kafka.common.resource.ResourcePatternFilter;
+import org.apache.kafka.common.resource.ResourceType;
+import org.apache.kafka.server.authorizer.Authorizer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Option;
+import scala.collection.JavaConverters;
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.stream.Collectors;
+
+public class MirrorSyncConfig {
+    private static final Logger log = LoggerFactory.getLogger(MirrorSyncConfig.class);
+    private static final ResourcePatternFilter ANY_TOPIC = new ResourcePatternFilter(ResourceType.TOPIC, null, PatternType.ANY);
+    private static final AclBindingFilter ANY_TOPIC_ACL = new AclBindingFilter(ANY_TOPIC, AccessControlEntryFilter.ANY);
+    private final Map<String, AdminClient> adminClientMap = new ConcurrentHashMap<>();
+    private static final String DIDI_CONFIG_PREFIX = "didi.ha";
+    private final AdminManager adminManager;
+    private final Option<Authorizer> authorizer;
+    private HashMap<String, List<AclBinding>> cachedAcls = new HashMap<>();
+
+    public MirrorSyncConfig(AdminManager adminManager, Option<Authorizer> authorizer) {
+        this.adminManager = adminManager;
+        this.authorizer = authorizer;
+    }
+
+    /**
+     * 根据mirrorTopic关联的cluster进行初始化adminClient
+     *
+     * @param clusterSet
+     */
+    public void initAdminClient(Set<String> clusterSet) {
+        clusterSet.forEach(clusterId -> {
+            if (adminClientMap.get(clusterId) == null) {
+                Properties properties = ClusterConfigManager.getConfigs(clusterId);
+                if (!properties.isEmpty()) {
+                    try {
+                        adminClientMap.put(clusterId, AdminClient.create(properties));
+                    } catch (Exception e) {
+                        log.error("can not init adminClient for cluster {} with {} ", clusterId, properties, e);
+                    }
+                }
+            }
+        });
+        //释放掉无关联的adminClient
+        Set<String> removedSet = adminClientMap.keySet().stream().filter(y -> !clusterSet.contains(y)).collect(Collectors.toSet());
+        removedSet.forEach(x -> {
+            releaseAdminClient(x, adminClientMap.get(x));
+        });
+    }
+
+    public void close() {
+        adminClientMap.forEach(this::releaseAdminClient);
+    }
+
+    private synchronized void releaseAdminClient(String clusterId, AdminClient client) {
+        try {
+            if (client != null) {
+                client.close();
+            }
+        } catch (Exception e) {
+            log.error("release cluster {} adminClient failure ", clusterId, e);
+        } finally {
+            adminClientMap.remove(clusterId);
+        }
+    }
+
+    public void syncTopicsAcls(List<MirrorTopic> allTopics) {
+        if (!authorizer.isEmpty()) {
+            Authorizer auth = authorizer.get();
+            Map<String, List<MirrorTopic>> remoteTopic = allTopics.stream().collect(Collectors.groupingBy(MirrorTopic::remoteCluster));
+            remoteTopic.forEach((key, value) -> {
+                AdminClient client = adminClientMap.get(key);
+                if (client != null) {
+                    Map<String, MirrorTopic> topics = value.stream().collect(Collectors.toMap(MirrorTopic::remoteTopic, x -> x));
+                    syncMirrorTopicsAcl(client, key, topics, auth);
+                }
+            });
+        }
+    }
+
+    public void syncTopicConfigs(List<MirrorTopic> allTopics) {
+        Map<String, List<MirrorTopic>> remoteTopic = allTopics.stream().collect(Collectors.groupingBy(MirrorTopic::remoteCluster));
+        remoteTopic.forEach((key, value) -> {
+            AdminClient client = adminClientMap.get(key);
+            if (client != null) {
+                Map<String, MirrorTopic> topics = value.stream().collect(Collectors.toMap(MirrorTopic::remoteTopic, x -> x));
+                syncMirrorTopicsConfig(client, key, topics);
+            }
+        });
+    }
+
+    private void syncMirrorTopicsAcl(AdminClient client, String clusterId, Map<String, MirrorTopic> topics, Authorizer auth) {
+        try {
+            Collection<AclBinding> result = client.describeAcls(ANY_TOPIC_ACL).values().get(30, TimeUnit.SECONDS);
+            List<AclBinding> remoteBindings = result.stream()
+                    .filter(x -> x.pattern().resourceType() == ResourceType.TOPIC)
+                    .filter(x -> x.pattern().patternType() == PatternType.LITERAL)
+                    .filter(x -> topics.containsKey(x.pattern().name()))
+                    .map(x -> new AclBinding(new ResourcePattern(x.pattern().resourceType(), topics.get(x.pattern().name()).localTopic(), x.pattern().patternType()), x.entry()))
+                    .collect(Collectors.toList());
+            List<AclBinding> cacheClusterAcl = cachedAcls.get(clusterId);
+            if (cacheClusterAcl != null) {
+                //计算出远程集群修改过或已经删除的ACL信息，并同步删除本地ACL信息
+                List<AclBinding> diffAcl = new ArrayList<>(cacheClusterAcl);
+                diffAcl.removeAll(remoteBindings);
+                List<AclBindingFilter> remoteAclBindingFilters = diffAcl.stream().map(AclBinding::toFilter).collect(Collectors.toList());
+                if (!remoteAclBindingFilters.isEmpty()) {
+                    auth.deleteAcls(null, remoteAclBindingFilters);
+                }
+            }
+            if (!remoteBindings.isEmpty() && !remoteBindings.equals(cacheClusterAcl)) {
+                auth.createAcls(null, remoteBindings);
+            }
+            //缓存当前的ACL信息
+            cachedAcls.put(clusterId, remoteBindings);
+        } catch (Exception e) {
+            log.error("sync cluster {} topic {} acls config", clusterId, topics.keySet(), e);
+        }
+    }
+
+    private void syncMirrorTopicsConfig(AdminClient client, String clusterId, Map<String, MirrorTopic> topics) {
+        try {
+            Set<ConfigResource> resources = topics.values().stream().filter(topic -> !topic.remoteTopic().equals(""))
+                    .map(x -> new ConfigResource(ConfigResource.Type.TOPIC, x.remoteTopic()))
+                    .collect(Collectors.toSet());
+            DescribeConfigsOptions describeOptions = new DescribeConfigsOptions().includeSynonyms(true);
+            //获取远程集群的Topic配置
+            Map<String, Config> remoteConfigs = client.describeConfigs(resources, describeOptions).all().get(30, TimeUnit.SECONDS).entrySet().stream()
+                    .collect(Collectors.toMap(x -> topics.get(x.getKey().name()).localTopic(), Map.Entry::getValue));
+            scala.collection.mutable.Map<ConfigResource, Option<scala.collection.Set<String>>> localConfigNames = new scala.collection.mutable.HashMap<>();
+            //筛选出用户动态改变的Topic配置
+            Map<String, Map<String, String>> remoteConfigMap = new HashMap<>();
+            for (Map.Entry<String, Config> entry : remoteConfigs.entrySet()) {
+                List<String> configNames = new ArrayList<>();
+                for (ConfigEntry cfg : entry.getValue().entries()) {
+                    if (cfg.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG && !cfg.name().startsWith(DIDI_CONFIG_PREFIX)) {
+                        configNames.add(cfg.name());
+                        remoteConfigMap.computeIfAbsent(entry.getKey(), v -> new HashMap<>()).put(cfg.name(), cfg.value());
+                    }
+                }
+                if (!configNames.isEmpty()) {
+                    localConfigNames.put(new ConfigResource(ConfigResource.Type.TOPIC, entry.getKey()), Option.apply(JavaConverters.asScalaBuffer(configNames).toSet()));
+                }
+            }
+            //根据远程集群的Topic配置项来获取本地集群的Topic配置
+            scala.collection.Map<ConfigResource, DescribeConfigsResponse.Config> localConfigs = adminManager.describeConfigs(localConfigNames, false);
+            scala.collection.mutable.Map<ConfigResource, scala.collection.immutable.List<AlterConfigOp>> alterConfig = new scala.collection.mutable.HashMap<>();
+            for (Map.Entry<String, Config> entry : remoteConfigs.entrySet()) {
+                Config remoteConfig = entry.getValue();
+                String topic = entry.getKey();
+                Map<String, String> remoteTopicConfig = remoteConfigMap.get(topic);
+                List<AlterConfigOp> newEntry = new ArrayList<>();
+                DescribeConfigsResponse.Config local = localConfigs.get(new ConfigResource(ConfigResource.Type.TOPIC, topic)).get();
+                local.entries().forEach(e -> {
+                    ConfigEntry configEntry = remoteConfig.get(e.name());
+                    //筛选出本地态动配置
+                    if (configEntry != null && e.source() == DescribeConfigsResponse.ConfigSource.TOPIC_CONFIG) {
+                        //筛选出远程与本地不一致的配置项
+                        if (!configEntry.value().equals(e.value())) {
+                            newEntry.add(new AlterConfigOp(new ConfigEntry(configEntry.name(), configEntry.value()), AlterConfigOp.OpType.SET));
+                        }
+                        //删除远程与本地相同配置项,目的是把远程新增同步到本地
+                        remoteTopicConfig.remove(e.name());
+                    }
+                });
+                //增加远程新配置项
+                remoteTopicConfig.forEach((k, v) -> newEntry.add(new AlterConfigOp(new ConfigEntry(k, v), AlterConfigOp.OpType.SET)));
+                if (!newEntry.isEmpty()) {
+                    alterConfig.put(new ConfigResource(ConfigResource.Type.TOPIC, topic), JavaConverters.asScalaBuffer(newEntry).toList());
+                }
+            }
+            //更新本地Topic配置
+            if (!alterConfig.isEmpty()) {
+                adminManager.incrementalAlterConfigs(alterConfig, false);
+            }
+        } catch (Exception e) {
+            log.error("sync cluster {} topic {} config", clusterId, topics.keySet(), e);
+        }
+    }
+
+    /**
+     * 如果zk节点配置发生变化只重建旧的adminClient,新增的不会创建
+     *
+     * @param clusterId
+     */
+    public void reloadAdminClient(String clusterId) {
+        AdminClient client = adminClientMap.get(clusterId);
+        if (client != null) {
+            Properties properties = ClusterConfigManager.getConfigs(clusterId);
+            try {
+                releaseAdminClient(clusterId, client);
+                if (!properties.isEmpty()) {
+                    adminClientMap.put(clusterId, AdminClient.create(properties));
+                }
+            } catch (Exception e) {
+                log.error("can not reload adminClient for cluster {} with {} ", clusterId, properties, e);
+            }
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/manager/ClusterConfigManager.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.config.manager;
+
+import com.didichuxing.datachannel.kafka.config.HAClusterConfig;
+import kafka.coordinator.mirror.MirrorCoordinator;
+import kafka.server.ConfigType;
+import kafka.server.ReplicaManager;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.errors.InvalidConfigurationException;
+import org.apache.kafka.common.network.ListenerName;
+import org.apache.kafka.common.requests.RequestContext;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class ClusterConfigManager {
+    public static final String GATEWAY_CLUSTER_ID = "GW";
+
+    private static final Logger log = LoggerFactory.getLogger(ClusterConfigManager.class);
+
+    private static Map<String, Properties> didiHAClusterConfigs = new HashMap<>();
+
+    private static ConfigDef DIDI_HA_CLUSTER_CONFIGS = HAClusterConfig.configDef();
+
+    private ReplicaManager replicaManager;
+
+    private MirrorCoordinator mirrorCoordinator;
+
+    public ClusterConfigManager(ReplicaManager replicaManager, MirrorCoordinator mirrorCoordinator) {
+        this.replicaManager = replicaManager;
+        this.mirrorCoordinator = mirrorCoordinator;
+    }
+
+    public static void validateConfigs(Properties configs) {
+        for (String key : configs.stringPropertyNames()) {
+            if (!HAClusterConfig.configNames().contains(key))
+                throw new InvalidConfigurationException(String.format("Unknown cluster config name: %s", key));
+        }
+        Map<String, Object> realConfigs = DIDI_HA_CLUSTER_CONFIGS.parse(configs);
+        for (String key : configs.stringPropertyNames()) {
+            if (DIDI_HA_CLUSTER_CONFIGS.configKeys().get(key).validator != null)
+                DIDI_HA_CLUSTER_CONFIGS.configKeys().get(key).validator.ensureValid(key, realConfigs.get(key));
+        }
+    }
+
+    public void configure(String cluster, Properties configs) {
+        if (configs.isEmpty()) {
+            if (didiHAClusterConfigs.containsKey(cluster)) {
+                didiHAClusterConfigs.remove(cluster);
+                log.info("clean configs for cluster:" + cluster);
+            }
+            return;
+        }
+        if (didiHAClusterConfigs.put(cluster, configs) != null) {
+            this.replicaManager.reloadRemoteCluster(cluster);
+            this.mirrorCoordinator.onConfigChanged(ConfigType.HACluster(), cluster);
+        }
+        log.info("set or update configs for cluster: {}, configs of this cluster: {}", cluster, getConfigs(cluster));
+    }
+
+    public static Object getConfig(String clusterId, String key) {
+        if (!didiHAClusterConfigs.containsKey(clusterId)) return null;
+        return didiHAClusterConfigs.get(clusterId).get(key);
+    }
+
+    public static Properties getConfigs(String clusterId) {
+        if (!didiHAClusterConfigs.containsKey(clusterId)) return new Properties();
+        return didiHAClusterConfigs.get(clusterId);
+    }
+
+    public static boolean existsHACluster(String clusterId) {
+        return didiHAClusterConfigs.containsKey(clusterId);
+    }
+
+    public static HAClusterConfig getHAClusterConfig(String clusterId) {
+        Properties props = getConfigs(clusterId);
+        log.info("Get mirror config for cluster {}} configs {}", clusterId, props);
+        return new HAClusterConfig(props);
+    }
+
+    public static Collection<Node> appendWithGatewayNodes(RequestContext context, Collection<Node> brokers) {
+        if (context.securityProtocol == SecurityProtocol.PLAINTEXT) {
+            return brokers;
+        }
+        if (!UserConfigManager.existsHAUser(context)) {
+            return brokers;
+        }
+        List<String> urls = getHAClusterConfig(GATEWAY_CLUSTER_ID)
+                .getList(HAClusterConfig.BOOTSTRAP_SERVERS_CONFIG);
+        List<Node> nodes = new ArrayList<>(brokers);
+        int id = -10;
+        for (String url : urls) {
+            nodes.add(new Node(id, Utils.getHost(url),  Utils.getPort(url)));
+            id --;
+        }
+        return nodes;
+    }
+
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/manager/GroupConfigManager.java

@@ -0,0 +1,70 @@
+package com.didichuxing.datachannel.kafka.config.manager;
+
+import com.didichuxing.datachannel.kafka.config.HAGroupConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.errors.InvalidConfigurationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * @author 杨阳
+ * @date {2022/6/1}
+ */
+public class GroupConfigManager {
+
+    private static final Logger log = LoggerFactory.getLogger(GroupConfigManager.class);
+
+    private static Map<String, Properties> didiHAGroupConfigs = new HashMap<>();
+
+    private static ConfigDef DIDI_HA_GROUP_CONFIGS = HAGroupConfig.configDef();
+
+    public GroupConfigManager() {
+
+    }
+
+    public static void validateConfigs(Properties configs) {
+        for (String key : configs.stringPropertyNames()) {
+            if (!HAGroupConfig.configNames().contains(key))
+                throw new InvalidConfigurationException(String.format("Unknown group config name: %s", key));
+        }
+        Map<String, Object> realConfigs = DIDI_HA_GROUP_CONFIGS.parse(configs);
+        for (String key : configs.stringPropertyNames()) {
+            if (DIDI_HA_GROUP_CONFIGS.configKeys().get(key).validator != null)
+                DIDI_HA_GROUP_CONFIGS.configKeys().get(key).validator.ensureValid(key, realConfigs.get(key));
+        }
+
+    }
+
+    public void configure(String group, Properties configs) {
+        if (configs.isEmpty()) {
+            if (didiHAGroupConfigs.containsKey(group)) {
+                didiHAGroupConfigs.remove(group);
+                log.info("clean configs for group:" + group);
+            }
+            return;
+        }
+        if (didiHAGroupConfigs.put(group, configs) != null) {
+            // reload changes
+        }
+        log.info("set or update configs for group: {}, configs of this group: {}", group, getConfigs(group));
+    }
+
+    public static Object getConfig(String group, String key) {
+        if (!didiHAGroupConfigs.containsKey(group)) return null;
+        return didiHAGroupConfigs.get(group).get(key);
+    }
+
+    public static Properties getConfigs(String group) {
+        if (!didiHAGroupConfigs.containsKey(group)) return new Properties();
+        return didiHAGroupConfigs.get(group);
+    }
+
+    public static HAGroupConfig getHAGroupConfig(String group) {
+        Properties props = getConfigs(group);
+        return new HAGroupConfig(props);
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/manager/TopicConfigManager.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.config.manager;
+
+import com.didichuxing.datachannel.kafka.config.HATopicConfig;
+import kafka.server.MirrorTopic;
+import kafka.server.ReplicaManager;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.errors.InvalidConfigurationException;
+import org.apache.kafka.common.internals.Topic;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class TopicConfigManager {
+
+    private static final ConfigDef DIDI_HA_TOPIC_CONFIGS = HATopicConfig.configDef();
+    private static final Map<String, MirrorTopic> mirrorTopics = new HashMap<>();
+    private static final Set<String> clusterRelatedConfigs = new HashSet<>();
+
+    static {
+        clusterRelatedConfigs.add(HATopicConfig.DIDI_HA_REMOTE_CLUSTER);
+    }
+
+    public static final Logger log = LoggerFactory.getLogger(TopicConfigManager.class);
+
+    private final ReplicaManager replicaManager;
+
+    public TopicConfigManager(ReplicaManager replicaManager) {
+        this.replicaManager = replicaManager;
+    }
+
+    public static void validateConfigs(Properties configs) {
+        for (String key : configs.stringPropertyNames()) {
+            if (!HATopicConfig.configNames().contains(key))
+                throw new InvalidConfigurationException(String.format("Unknown ha-topic config name: %s", key));
+        }
+        Map<String, Object> realConfigs = DIDI_HA_TOPIC_CONFIGS.parse(configs);
+        for (String key : configs.stringPropertyNames()) {
+            if (DIDI_HA_TOPIC_CONFIGS.configKeys().get(key).validator != null)
+                DIDI_HA_TOPIC_CONFIGS.configKeys().get(key).validator.ensureValid(key, realConfigs.get(key));
+        }
+
+    }
+
+    //某Topic的配置中是否有对集群cluster的依赖
+    public static Boolean ifConfigsClusterRelated(Properties configs, String cluster) {
+        for (String clusterRelatedConfig : clusterRelatedConfigs) {
+            if (configs.containsKey(clusterRelatedConfig) && Objects.equals(configs.getProperty(clusterRelatedConfig), cluster))
+                return true;
+        }
+        return false;
+    }
+
+    public void configure(String topicName, Properties configs) {
+        if (Topic.isInternal(topicName) && !Objects.equals(topicName, Topic.GROUP_METADATA_TOPIC_NAME))
+            return;
+        //the remoteTopic cannot be any of InternalTopics
+        Map<String, Object> realConfiges = DIDI_HA_TOPIC_CONFIGS.parse(configs);
+        if (configs.containsKey(HATopicConfig.DIDI_HA_REMOTE_CLUSTER)) {
+            String remoteTopicName = (String) configs.getOrDefault(HATopicConfig.DIDI_HA_REMOTE_TOPIC, topicName);
+            Boolean syncTopicPartitions = (Boolean) realConfiges.getOrDefault(HATopicConfig.DIDI_HA_SYNC_TOPIC_PARTITIONS_ENABLED, false);
+            Boolean syncTopicConfigs = (Boolean) realConfiges.getOrDefault(HATopicConfig.DIDI_HA_SYNC_TOPIC_CONFIGS_ENABLED, false);
+            Boolean syncTopicAcls = (Boolean) realConfiges.getOrDefault(HATopicConfig.DIDI_HA_SYNC_TOPIC_ACLS_ENABLED, false);
+            MirrorTopic mirrorTopic = new MirrorTopic(remoteTopicName,
+                    configs.getProperty(HATopicConfig.DIDI_HA_REMOTE_CLUSTER),
+                    topicName,
+                    syncTopicPartitions,
+                    syncTopicConfigs,
+                    syncTopicAcls);
+            replicaManager.addMirrorTopics(mirrorTopic);
+            mirrorTopics.put(topicName, mirrorTopic);
+        } else {
+            replicaManager.removeMirrorTopicsByLocalTopic(topicName);
+            mirrorTopics.remove(topicName);
+        }
+    }
+
+    public static boolean isMirrorTopicByLocal(String topicName) {
+        if (Topic.GROUP_METADATA_TOPIC_NAME.equals(topicName)) {
+            return false;
+        }
+        return mirrorTopics.containsKey(topicName);
+    }
+
+}

core/src/main/java/com/didichuxing/datachannel/kafka/config/manager/UserConfigManager.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.config.manager;
+
+import com.didichuxing.datachannel.kafka.config.HAUserConfig;
+import kafka.server.DynamicConfig;
+import org.apache.kafka.common.requests.RequestContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.function.Supplier;
+
+public class UserConfigManager {
+    private static final Logger log = LoggerFactory.getLogger(UserConfigManager.class);
+    private static final Map<String, Properties> didiHAUserConfigs = new HashMap<>();
+    private static final Set<String> clusterRelatedConfigs = new HashSet<>();
+
+    static {
+        clusterRelatedConfigs.add(HAUserConfig.DIDI_HA_ACTIVE_CLUSTER_CONFIG);
+    }
+
+    public void configure(String user, Properties configs) {
+        Map<String, Object> realConfigs = DynamicConfig.getUserConfigs().parse(configs);
+        Properties realKMConfigs = new Properties();
+        for (String configName : HAUserConfig.configNames()) {
+            if (Objects.nonNull(realConfigs.get(configName)))
+                realKMConfigs.put(configName, realConfigs.get(configName));
+        }
+        if (realKMConfigs.isEmpty()) {
+            if (didiHAUserConfigs.containsKey(user)) {
+                didiHAUserConfigs.remove(user);
+                log.info("Remove configs for user: {}", user);
+            }
+        } else {
+            didiHAUserConfigs.put(user, realKMConfigs);
+            log.info("Set configs for user {} : {}", user, realKMConfigs);
+        }
+    }
+
+    //某User的配置中是否有对集群cluster的依赖
+    public static Boolean ifConfigsClusterRelated(Properties configs, String cluster) {
+        for (String clusterRelatedConfig : clusterRelatedConfigs) {
+            if (configs.containsKey(clusterRelatedConfig) && Objects.equals(configs.getProperty(clusterRelatedConfig), cluster))
+                return true;
+        }
+        return false;
+    }
+
+    public static boolean existsHAUser(String key) {
+        return didiHAUserConfigs.containsKey(key)
+                && didiHAUserConfigs.get(key).containsKey(HAUserConfig.DIDI_HA_ACTIVE_CLUSTER_CONFIG);
+    }
+
+    public static boolean existsHAUser(RequestContext context) {
+        String identity1 = context.principal.getName() + "#" + context.clientId();
+        String identity2 = context.principal.getName();
+        return existsHAUser(identity1) || existsHAUser(identity2);
+    }
+
+    public static String activeCluster(RequestContext context, String defaultCluster) {
+        return activeCluster(context, defaultCluster, Optional::empty);
+    }
+
+    public static String activeCluster(RequestContext context, String defaultCluster, Supplier<Optional<String>> activeClusterForTopicCallback) {
+        String identity = context.principal.getName() + "#" + context.clientId();
+        if (!existsHAUser(identity)) {
+            Optional<String> topicCluster = activeClusterForTopicCallback.get();
+            if (topicCluster.isPresent()
+                    // 排除用户在切换中状态
+                    && !activeClusterByIdentity(context.principal.getName(), defaultCluster).equals("None")) {
+                return topicCluster.get();
+            }
+            identity = context.principal.getName();
+        }
+        return activeClusterByIdentity(identity, defaultCluster);
+    }
+
+    private static String activeClusterByIdentity(String identity, String defaultCluster) {
+        return getConfigs(identity).getProperty(HAUserConfig.DIDI_HA_ACTIVE_CLUSTER_CONFIG, defaultCluster);
+    }
+
+    public static Properties getConfigs(String user) {
+        if (!didiHAUserConfigs.containsKey(user)) return new Properties();
+        return didiHAUserConfigs.get(user);
+    }
+
+    public static HAUserConfig getHAUserConfig(String user) {
+        Properties props = getConfigs(user);
+        return new HAUserConfig(props);
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/AppIdHostTopicMetrics.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.MetricName;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.StringJoiner;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+public class AppIdHostTopicMetrics {
+    public static ConcurrentMap<String, AppIdHostTopicMetrics> allClientRequestMetrics = new ConcurrentHashMap<>();
+
+    private static final String group = "kafka.server";
+    private static final String type = "ClientRequestMetrics";
+
+    private final String NetRequestBytesPerSec = "NetRequestPerSecnd";
+    private final String ProduceRequestPerSec = "ProduceRequestPerSec";
+    private final String ProduceRequestBytesPerSec = "ProduceRequestBytesPerSec";
+    private final String FetchRequestPerSec = "FetchRequestPerSec";
+    private final String FetchRequestBytesPerSec = "FetchRequestBytesPerSec";
+
+    private final ExMeter netRequestInRate;
+    private final ExMeter produceRequestInRate;
+    private final ExMeter produceRequestBytesInRate;
+    private final ExMeter fetchRequestInRate;
+    private final ExMeter fetchRequestBytesOutRate;
+
+    public AppIdHostTopicMetrics(String topic, String appId, String ip) {
+        Map<String, String> metricsTags = new LinkedHashMap<>();
+        metricsTags.put("topic", topic);
+        metricsTags.put("appId", appId);
+        metricsTags.put("ip", ip);
+
+        MetricName metricName = KafkaExMetrics.createMetricsName(group, type, NetRequestBytesPerSec, metricsTags);
+        netRequestInRate = KafkaExMetrics.createMeter(metricName, "net request metrics", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ProduceRequestPerSec, metricsTags);
+        produceRequestInRate = KafkaExMetrics.createMeter(metricName, "produce request metrics", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ProduceRequestBytesPerSec, metricsTags);
+        produceRequestBytesInRate = KafkaExMetrics.createMeter(metricName, "produce request metrics", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, FetchRequestPerSec, metricsTags);
+        fetchRequestInRate = KafkaExMetrics.createMeter(metricName, "fetch request metrics", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, FetchRequestBytesPerSec, metricsTags);
+        fetchRequestBytesOutRate = KafkaExMetrics.createMeter(metricName, "fetch request metrics", TimeUnit.SECONDS);
+    }
+
+    public void close() {
+        KafkaExMetrics.removeMetrics(netRequestInRate);
+        KafkaExMetrics.removeMetrics(produceRequestInRate);
+        KafkaExMetrics.removeMetrics(produceRequestBytesInRate);
+        KafkaExMetrics.removeMetrics(fetchRequestInRate);
+        KafkaExMetrics.removeMetrics(fetchRequestBytesOutRate);
+    }
+
+    public static AppIdHostTopicMetrics getClientMetrics(String topic, String appId, String ip) {
+        String key = topic + "#" + appId + "#" + ip;
+        return allClientRequestMetrics.computeIfAbsent(key, (k) -> new AppIdHostTopicMetrics(topic, appId, ip));
+    }
+
+    public static void removeClientMetrics(String topic, String appId, String ip) {
+        String key = topic + "#" + appId + "#" + ip;
+        removeClientMetricsByKey(key);
+    }
+
+    public static void removeClientMetrics(String topic) {
+        var itr = allClientRequestMetrics.entrySet().iterator();
+        while (itr.hasNext()) {
+            var entry = itr.next();
+            String name = entry.getKey();
+            if (name.startsWith(topic.concat("#"))) {
+                AppIdHostTopicMetrics metrics = entry.getValue();
+                metrics.close();
+                itr.remove();
+            }
+        }
+    }
+
+    private static void removeClientMetricsByKey(String key) {
+        AppIdHostTopicMetrics metrics = allClientRequestMetrics.remove(key);
+        metrics.close();
+    }
+
+    public Meter netRequestInRate() {
+        return netRequestInRate.meter();
+    }
+
+    public Meter produceRequestInRate() {
+        return produceRequestInRate.meter();
+    }
+
+    public Meter produceRequestBytesInRate() {
+        return produceRequestBytesInRate.meter();
+    }
+
+    public Meter fetchRequestInRate() {
+        return fetchRequestInRate.meter();
+    }
+
+    public Meter fetchRequestBytesOutRate() {
+        return fetchRequestBytesOutRate.meter();
+    }
+
+    public static String getType() {
+        return type;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/AppIdTopicMetrics.java

@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.didichuxing.datachannel.kafka.util.KafkaUtils;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.MetricName;
+import kafka.server.BrokerTopicStats;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.StringJoiner;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+public class AppIdTopicMetrics {
+
+    private static final String group = "kafka.server";
+    private static final String type = "AppIdTopicMetrics";
+
+    public static ConcurrentMap<String, AppIdTopicMetrics> allMetrics = new ConcurrentHashMap<>();
+
+    private final ExMeter messagesInRate;
+    private final ExMeter bytesInRate;
+    private final ExMeter bytesOutRate;
+    private final ExMeter bytesRejectedRate;
+    private final ExMeter totalProduceRequestsRate;
+    private final ExMeter totalFetchRequestsRate;
+    private final ExMeter failedProduceRequestsRate;
+    private final ExMeter failedFetchRequestsRate;
+    private final ExMeter produceMessageConversionsRate;
+    private final ExMeter fetchMessageConversionsRate;
+
+    private final ExGauge<String> produceApiVersionGauge;
+    private final ExGauge<String> fetchApiVersionGauge;
+    private final ExGauge<String> recordCompressionGauge;
+
+    private String produceApiVersion = "";
+    private String fetchApiVersion = "";
+    private String recordCompression = "";
+
+    private final String topic;
+
+    public AppIdTopicMetrics(String topic, String appId) {
+        this.topic = topic;
+
+        Map<String, String> metricsTags = new LinkedHashMap<>();
+        metricsTags.put("topic", topic);
+        metricsTags.put("appId", appId);
+
+        MetricName metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.MessagesInPerSec(), metricsTags);
+        messagesInRate = KafkaExMetrics.createMeter(metricName, "message", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.BytesInPerSec(), metricsTags);
+        bytesInRate = KafkaExMetrics.createMeter(metricName, "bytes in", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.BytesOutPerSec(), metricsTags);
+        bytesOutRate = KafkaExMetrics.createMeter(metricName, "bytes out", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.BytesRejectedPerSec(), metricsTags);
+        bytesRejectedRate = KafkaExMetrics.createMeter(metricName, "bytes rejected", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.TotalProduceRequestsPerSec(), metricsTags);
+        totalProduceRequestsRate = KafkaExMetrics.createMeter(metricName, "requests", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.TotalFetchRequestsPerSec(), metricsTags);
+        totalFetchRequestsRate = KafkaExMetrics.createMeter(metricName, "requests", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.FailedProduceRequestsPerSec(), metricsTags);
+        failedProduceRequestsRate = KafkaExMetrics.createMeter(metricName, "requests", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.FailedFetchRequestsPerSec(), metricsTags);
+        failedFetchRequestsRate = KafkaExMetrics.createMeter(metricName, "requests", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.ProduceMessageConversionsPerSec(), metricsTags);
+        produceMessageConversionsRate = KafkaExMetrics.createMeter(metricName, "requests", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, BrokerTopicStats.FetchMessageConversionsPerSec(), metricsTags);
+        fetchMessageConversionsRate = KafkaExMetrics.createMeter(metricName, "requests", TimeUnit.SECONDS);
+
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, "ProduceApiVer", metricsTags);
+        produceApiVersionGauge = KafkaExMetrics.createGauge(metricName, new Gauge<>() {
+            @Override
+            public String value() {
+                return produceApiVersion;
+            }
+        });
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, "FetchApiVer", metricsTags);
+        fetchApiVersionGauge = KafkaExMetrics.createGauge(metricName, new Gauge<>() {
+            @Override
+            public String value() {
+                return fetchApiVersion;
+            }
+        });
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, "RecordCompression", metricsTags);
+        recordCompressionGauge = KafkaExMetrics.createGauge(metricName, new Gauge<>() {
+            @Override
+            public String value() {
+                return recordCompression;
+            }
+        });
+
+    }
+
+    public void close() {
+        KafkaExMetrics.removeMetrics(messagesInRate);
+        KafkaExMetrics.removeMetrics(bytesInRate);
+        KafkaExMetrics.removeMetrics(bytesOutRate);
+        KafkaExMetrics.removeMetrics(bytesRejectedRate);
+        KafkaExMetrics.removeMetrics(totalProduceRequestsRate);
+        KafkaExMetrics.removeMetrics(totalFetchRequestsRate);
+        KafkaExMetrics.removeMetrics(failedProduceRequestsRate);
+        KafkaExMetrics.removeMetrics(failedFetchRequestsRate);
+        KafkaExMetrics.removeMetrics(produceMessageConversionsRate);
+        KafkaExMetrics.removeMetrics(fetchMessageConversionsRate);
+        KafkaExMetrics.removeMetrics(produceApiVersionGauge);
+        KafkaExMetrics.removeMetrics(fetchApiVersionGauge);
+        KafkaExMetrics.removeMetrics(recordCompressionGauge);
+    }
+
+    public static AppIdTopicMetrics getAppIdTopicMetrics(String topic, String appId) {
+        String key = topic + "#" + appId;
+        return allMetrics.computeIfAbsent(key, (k) -> new AppIdTopicMetrics(topic, appId));
+    }
+
+    public static void removeAppIdTopicMetrics(String topic, String appId) {
+        String key = topic + "#" + appId;
+        AppIdTopicMetrics metrics = allMetrics.remove(key);
+        metrics.close();
+    }
+
+    public static void removeAppIdTopicMetrics(String topic) {
+        var itr = allMetrics.entrySet().iterator();
+        while (itr.hasNext()) {
+            var entry = itr.next();
+            String name = entry.getKey();
+            if (name.startsWith(topic.concat("#"))) {
+                AppIdTopicMetrics metrics = entry.getValue();
+                metrics.close();
+                itr.remove();
+            }
+        }
+    }
+
+    public void markMessagesInRate(long size) {
+        messagesInRate.meter().mark(size);
+    }
+
+    public void markBytesInRate(long size) {
+        bytesInRate.meter().mark(size);
+        AppIdHostTopicMetrics appIdHostTopicMetrics = KafkaExMetrics.appIdHostTopicMetrics(topic);
+        if (null != appIdHostTopicMetrics) {
+            appIdHostTopicMetrics.produceRequestBytesInRate().mark(size);
+        }
+    }
+
+    public void markBytesOutRate(long size) {
+        bytesOutRate.meter().mark(size);
+        AppIdHostTopicMetrics appIdHostTopicMetrics = KafkaExMetrics.appIdHostTopicMetrics(topic);
+        if (null != appIdHostTopicMetrics) {
+            appIdHostTopicMetrics.fetchRequestBytesOutRate().mark(size);
+        }
+    }
+
+    public void markBytesRejectedRate(long size) {
+        bytesRejectedRate.meter().mark(size);
+    }
+
+    public void markTotalProduceRequestsRate() {
+        totalProduceRequestsRate.meter().mark();
+        AppIdHostTopicMetrics appIdHostTopicMetrics = KafkaExMetrics.appIdHostTopicMetrics(topic);
+        if (null != appIdHostTopicMetrics) {
+            appIdHostTopicMetrics.produceRequestInRate().mark();
+        }
+    }
+
+    public void markTotalFetchRequestsRate() {
+        totalFetchRequestsRate.meter().mark();
+        AppIdHostTopicMetrics appIdHostTopicMetrics = KafkaExMetrics.appIdHostTopicMetrics(topic);
+        if (null != appIdHostTopicMetrics) {
+            appIdHostTopicMetrics.fetchRequestInRate().mark();
+        }
+    }
+
+    public void markFailedProduceRequestsRate() {
+        failedProduceRequestsRate.meter().mark();
+    }
+
+    public void markFailedFetchRequestsRate() {
+        failedFetchRequestsRate.meter().mark();
+    }
+
+    public Meter produceMessageConversionsRate() {
+        return produceMessageConversionsRate.meter();
+    }
+
+    public Meter fetchMessageConversionsRate() {
+        return fetchMessageConversionsRate.meter();
+    }
+
+    public void markProduceApiVersion(int version) {
+        this.produceApiVersion = KafkaUtils.apiVersionToKafkaVersion(0, version);
+        this.produceApiVersionGauge.mark();
+    }
+
+    public void markFetchApiVersion(int version) {
+        this.fetchApiVersion = KafkaUtils.apiVersionToKafkaVersion(1, version);
+        this.fetchApiVersionGauge.mark();
+    }
+
+    public void markRecordCompression(String recordCompression) {
+        this.recordCompression = recordCompression;
+        this.recordCompressionGauge.mark();
+    }
+
+    public static String getType() {
+        return type;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/DiskTopicPartitionMetrics.java

@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.core.MetricName;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.StringJoiner;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+public class DiskTopicPartitionMetrics {
+
+    private static final String group = "kafka.server";
+    private static final String type = "DiskMetrics";
+
+    public static ConcurrentMap<String, DiskTopicPartitionMetrics> allMetrics = new ConcurrentHashMap<>();
+
+    private final String ReplicRealDiskReadPerSec = "ReplicRealDiskReadPerSec";
+    private final String ConsumerRealDiskReadPerSec = "ConsumerRealDiskReadPerSec";
+    private final String ReassignmentRealDiskReadPerSec = "ReassignmentRealDiskReadPerSec";
+    private final String ReplicDiskReadPerSec = "ReplicDiskReadPerSec";
+    private final String ConsumerDiskReadPerSec = "ConsumerDiskReadPerSec";
+    private final String ReassignmentDiskReadPerSec = "ReassignmentDiskReadPerSec";
+    private final String ReplicDiskWritePerSec = "ReplicDiskWritePerSec";
+    private final String ProducerDiskWritePerSec = "ProducerDiskWritePerSec";
+    private final String ReassignmentDiskWritePerSec = "ReassignmentDiskWritePerSec";
+
+    private final  ExMeter replicaRealDiskReadtRate;
+    private final  ExMeter consumerRealDiskReadtRate;
+    private final  ExMeter reassignmentRealDiskReadtRate;
+    private final  ExMeter replicaDiskReadtRate;
+    private final  ExMeter consumerDiskReadtRate;
+    private final  ExMeter reassignmentDiskReadtRate;
+    private final  ExMeter replicaDiskWritetRate;
+    private final  ExMeter producerDiskWriteRate;
+    private final  ExMeter reassignmentDiskWriteRate;
+
+    private final String disk;
+    private final TopicPartition topicPartition;
+
+    public DiskTopicPartitionMetrics(String disk, TopicPartition topicPartition) {
+        this.disk = disk;
+        this.topicPartition = topicPartition;
+
+        Map<String, String> metricsTags = new LinkedHashMap<>();
+        metricsTags.put("disk", disk);
+        if (topicPartition != null) {
+            metricsTags.put("topic", topicPartition.toString());
+        }
+
+        MetricName metricName = KafkaExMetrics.createMetricsName(group, type, ReplicRealDiskReadPerSec, metricsTags);
+        replicaRealDiskReadtRate = KafkaExMetrics.createMeter(metricName, "replica read bytes from disk no in page cache", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ConsumerRealDiskReadPerSec, metricsTags);
+        consumerRealDiskReadtRate = KafkaExMetrics.createMeter(metricName, "consumer read bytes from disk no in page cache", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ReassignmentRealDiskReadPerSec, metricsTags);
+        reassignmentRealDiskReadtRate = KafkaExMetrics.createMeter(metricName, "reassignment read bytes from disk no in page cache ", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ReplicDiskReadPerSec, metricsTags);
+        replicaDiskReadtRate = KafkaExMetrics.createMeter(metricName, "replica read bytes from disk", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ConsumerDiskReadPerSec, metricsTags);
+        consumerDiskReadtRate = KafkaExMetrics.createMeter(metricName, "consumer read bytes from disk", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ReassignmentDiskReadPerSec, metricsTags);
+        reassignmentDiskReadtRate = KafkaExMetrics.createMeter(metricName, "reassignment read bytes from disk", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ReplicDiskWritePerSec, metricsTags);
+        replicaDiskWritetRate = KafkaExMetrics.createMeter(metricName, "replica write bytes to disk", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ProducerDiskWritePerSec, metricsTags);
+        producerDiskWriteRate = KafkaExMetrics.createMeter(metricName, "producer write bytes to disk", TimeUnit.SECONDS);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, ReassignmentDiskWritePerSec, metricsTags);
+        reassignmentDiskWriteRate = KafkaExMetrics.createMeter(metricName, "reassignment write bytes to disk", TimeUnit.SECONDS);
+    }
+
+    public void close() {
+        KafkaExMetrics.removeMetrics(replicaRealDiskReadtRate);
+        KafkaExMetrics.removeMetrics(consumerRealDiskReadtRate);
+        KafkaExMetrics.removeMetrics(reassignmentRealDiskReadtRate);
+        KafkaExMetrics.removeMetrics(replicaDiskReadtRate);
+        KafkaExMetrics.removeMetrics(consumerDiskReadtRate);
+        KafkaExMetrics.removeMetrics(reassignmentDiskReadtRate);
+        KafkaExMetrics.removeMetrics(replicaDiskWritetRate);
+        KafkaExMetrics.removeMetrics(producerDiskWriteRate);
+        KafkaExMetrics.removeMetrics(reassignmentDiskWriteRate);
+    }
+
+    public static DiskTopicPartitionMetrics getDiskTopicMetrics(String disk) {
+        return allMetrics.computeIfAbsent(disk, (k) -> new DiskTopicPartitionMetrics(disk, null));
+    }
+
+    public static DiskTopicPartitionMetrics getDiskTopicMetrics(String disk, TopicPartition topicPartition) {
+        String key = topicPartition + "#" + disk;
+        return allMetrics.computeIfAbsent(key, (k) -> new DiskTopicPartitionMetrics(disk, topicPartition));
+    }
+
+    /*
+        role : 0 consumer, 1 replica, -1 reassignment
+     */
+    public void updateReadMetrics(TopicPartition topicPartition, int size, int role, boolean cached) {
+        DiskTopicPartitionMetrics topicPartitionMetrics = KafkaExMetrics.diskTopicPartitionMetrics(this.disk, topicPartition);
+        switch (role) {
+            case -1:
+                reassignmentDiskReadtRate.meter().mark(size);
+                if (!cached) reassignmentRealDiskReadtRate.meter().mark(size);
+                if (null != topicPartitionMetrics) {
+                    topicPartitionMetrics.reassignmentDiskReadtRate.meter().mark(size);
+                    if (!cached) topicPartitionMetrics.reassignmentRealDiskReadtRate.meter().mark(size);
+                }
+                break;
+
+            case 0:
+                consumerDiskReadtRate.meter().mark(size);
+                if (!cached) consumerRealDiskReadtRate.meter().mark(size);
+                if (null != topicPartitionMetrics) {
+                    topicPartitionMetrics.consumerDiskReadtRate.meter().mark(size);
+                    if (!cached) topicPartitionMetrics.consumerRealDiskReadtRate.meter().mark(size);
+                }
+                break;
+
+            case 1:
+                replicaDiskReadtRate.meter().mark(size);
+                if (!cached) replicaRealDiskReadtRate.meter().mark(size);
+                if (null != topicPartitionMetrics) {
+                    topicPartitionMetrics.replicaDiskReadtRate.meter().mark(size);
+                    if (!cached) topicPartitionMetrics.replicaRealDiskReadtRate.meter().mark(size);
+                }
+                break;
+
+            default:
+        }
+    }
+
+    public void updateWriteMetrics(TopicPartition topicPartition, int size, int role) {
+        DiskTopicPartitionMetrics topicPartitionMetrics = KafkaExMetrics.diskTopicPartitionMetrics(this.disk, topicPartition);
+        switch (role) {
+            case -1:
+                reassignmentDiskWriteRate.meter().mark(size);
+                if (null != topicPartitionMetrics) {
+                    topicPartitionMetrics.reassignmentDiskWriteRate.meter().mark(size);
+                }
+                break;
+
+            case 0:
+                producerDiskWriteRate.meter().mark(size);
+                if (null != topicPartitionMetrics) {
+                    topicPartitionMetrics.producerDiskWriteRate.meter().mark(size);
+                }
+                break;
+
+            case 1:
+                replicaDiskWritetRate.meter().mark(size);
+                if (null != topicPartitionMetrics) {
+                    topicPartitionMetrics.replicaDiskWritetRate.meter().mark(size);
+                }
+                break;
+
+            default:
+        }
+    }
+
+    public static void removeMetrics(String disk, TopicPartition topicPartition) {
+        if (topicPartition == null) {
+            DiskTopicPartitionMetrics metrics = allMetrics.remove(disk);
+            if (null != metrics) {
+                metrics.close();
+            }
+        }
+        String key = topicPartition + "#" + disk;
+        DiskTopicPartitionMetrics metrics = allMetrics.remove(key);
+        if (null != metrics) {
+            metrics.close();
+        }
+    }
+
+    public static void removeMetrics(String topic) {
+        var itr = allMetrics.entrySet().iterator();
+        while (itr.hasNext()) {
+            var entry = itr.next();
+            String name = entry.getKey();
+            if (name.startsWith(topic.concat("-"))) {
+                DiskTopicPartitionMetrics metrics = entry.getValue();
+                metrics.close();
+                itr.remove();
+            }
+        }
+    }
+
+    public static String getType() {
+        return type;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/ExGauge.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.MetricName;
+
+public class ExGauge<T> extends ExMetrics {
+    private Gauge<T> _gauge;
+    private Gauge<T> gauge;
+
+    public ExGauge(MetricName metricName, Gauge<T> gauge) {
+        super(metricName);
+        this._gauge = gauge;
+    }
+
+    public void mark() {
+        if (gauge != null) return;
+
+        synchronized (this) {
+            if (gauge == null) {
+                this.gauge = Metrics.defaultRegistry().newGauge(name, this._gauge);
+            }
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/ExHistogram.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+
+public class ExHistogram extends ExMetrics {
+    private Histogram histogram;
+    private boolean biased;
+
+    public ExHistogram(MetricName metricName, boolean biased) {
+        super(metricName);
+        this.biased = biased;
+    }
+
+    public Histogram histogram() {
+        if (this.histogram != null) return histogram;
+
+        synchronized (this) {
+            if (this.histogram != null) return histogram;
+            this.histogram = Metrics.defaultRegistry().newHistogram(name, biased);
+            return histogram;
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/ExMeter.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.MetricName;
+
+import java.util.concurrent.TimeUnit;
+
+public class ExMeter extends ExMetrics {
+    private Meter meter;
+    private final String describe;
+    private final TimeUnit timeUnit;
+
+    public ExMeter(MetricName metricName, String describe, TimeUnit timeUnit) {
+        super(metricName);
+        this.describe = describe;
+        this.timeUnit = timeUnit;
+    }
+
+    public Meter meter() {
+        if (this.meter != null) return meter;
+
+        synchronized (this) {
+            if (this.meter != null) return meter;
+            this.meter = Metrics.defaultRegistry().newMeter(name, describe, timeUnit);
+            return meter;
+        }
+    }
+
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/ExMetrics.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.core.MetricName;
+
+public class ExMetrics {
+    protected final MetricName name;
+
+    public ExMetrics(MetricName name) {
+        this.name = name;
+    }
+
+    public MetricName getName() {
+        return name;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/KafkaExMetrics.java

@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.didichuxing.datachannel.kafka.server.OSMetrics;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.MetricName;
+import joptsimple.internal.Strings;
+import kafka.network.RequestChannel;
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.TopicPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class KafkaExMetrics implements Configurable {
+
+    private static final Logger log = LoggerFactory.getLogger(KafkaExMetrics.class);
+
+    private static ConcurrentHashMap<String, Rule> metricsRules = new ConcurrentHashMap<>();
+    private static OSMetrics osMetrics;
+
+    public static void init(org.apache.kafka.common.metrics.Metrics metrics, boolean enableAll,
+                            ScheduledExecutorService executorService) throws Exception {
+        osMetrics = new OSMetrics(metrics, executorService);
+        if (enableAll) {
+            new KafkaExMetrics().configure(Map.of("*", "true"));
+        }
+    }
+
+    public static MetricName createMetricsName(String group, String type, String name, Map<String, String> tags) {
+        StringBuilder builder = new StringBuilder();
+        builder.append(group);
+        builder.append(":type=");
+        builder.append(type);
+        builder.append(",name=");
+        builder.append(name);
+
+        Map<String, String> sorted = new TreeMap<String, String>(tags);
+        sorted.forEach((k, v) -> {
+            builder.append(String.format(",%s=%s", k, v.replaceAll("[,=:]", ".")));
+        });
+        return new MetricName(group, type, name, null, builder.toString());
+    }
+
+    public static TopicRequestMetrics fetchRequestMetrics(String topic) {
+        boolean matched = matchRules(TopicRequestMetrics.getType(), topic);
+        if (!matched) return null;
+        return TopicRequestMetrics.getFetchRequestMetrics(topic);
+    }
+
+    public static TopicRequestMetrics produceRequestMetrics(String topic) {
+        boolean matched = matchRules(TopicRequestMetrics.getType(), topic);
+        if (!matched) return null;
+        return TopicRequestMetrics.getProduceRequestMetrics(topic);
+    }
+
+    public static AppIdTopicMetrics appIdTopicMetrics(String topic) {
+        boolean matched = matchRules(AppIdTopicMetrics.getType(), topic);
+        if (!matched) return null;
+        String appId = RequestChannel.currentSession().getUsername();
+        return AppIdTopicMetrics.getAppIdTopicMetrics(topic, appId);
+    }
+
+    public static AppIdHostTopicMetrics appIdHostTopicMetrics(String topic) {
+        boolean matched = matchRules(AppIdHostTopicMetrics.getType(), topic);
+        if (!matched) return null;
+        String appId = RequestChannel.currentSession().getUsername();
+        String host = RequestChannel.currentSession().getHostAddress();
+        return AppIdHostTopicMetrics.getClientMetrics(topic, appId, host);
+    }
+
+    public static DiskTopicPartitionMetrics diskTopicPartitionMetrics(String disk) {
+        return DiskTopicPartitionMetrics.getDiskTopicMetrics(disk);
+    }
+
+    public static DiskTopicPartitionMetrics diskTopicPartitionMetrics(String disk, TopicPartition topicPartition) {
+        boolean matched = matchRules(DiskTopicPartitionMetrics.getType(), topicPartition.topic());
+        if (!matched) return null;
+        return DiskTopicPartitionMetrics.getDiskTopicMetrics(disk, topicPartition);
+    }
+
+    public static ExMeter createMeter(MetricName name, String describe, TimeUnit timeUnit) {
+        return new ExMeter(name, describe, timeUnit);
+    }
+
+    public static ExHistogram createHistogram(MetricName name, boolean biased) {
+        return new ExHistogram(name, biased);
+    }
+
+    public static <T> ExGauge<T> createGauge(MetricName name, Gauge<T> gauge) {
+        return new ExGauge<T>(name, gauge);
+    }
+
+    @Override
+    public void configure(Map<String, ?> configs) {
+        if (configs.isEmpty()) return;
+        StringBuilder builder = new StringBuilder();
+        ConcurrentHashMap<String, Rule> configRules = new ConcurrentHashMap<>();
+        for (var entry: configs.entrySet()) {
+            String name = entry.getKey();
+            boolean allow = false;
+            try {
+                allow = Boolean.parseBoolean((String) entry.getValue());
+            } catch (Exception ignored) {
+            }
+            Rule rule = new Rule(allow, false);
+            configRules.put(name, rule);
+            builder.append(String.format("\n\t%s=%b", name, allow));
+        }
+        log.info("KafkaExMetrics config is:{}", builder.toString());
+
+        ConcurrentHashMap<String, Rule> oldiMetricsRules = metricsRules;
+        metricsRules = configRules;
+        for (var entry : oldiMetricsRules.entrySet()) {
+            String[] keys = entry.getKey().split("\\.");
+            if (keys.length == 1 || keys[1].equals("*")) {
+                continue;
+            }
+            boolean matched = matchRules(keys[0], keys[1]);
+            if (!matched) {
+                switch (keys[0]) {
+                    case "TopicRequestMetrics":
+                        TopicRequestMetrics.removeRequestMetrics(keys[1]);
+                        break;
+                    case "AppIdTopicMetrics":
+                        AppIdTopicMetrics.removeAppIdTopicMetrics(keys[1]);
+                        break;
+                    case "ClientRequestMetrics":
+                        AppIdHostTopicMetrics.removeClientMetrics(keys[1]);
+                        break;
+                    case "DiskMetrics":
+                        DiskTopicPartitionMetrics.removeMetrics(keys[1]);
+                        break;
+                }
+            }
+        }
+    }
+
+    public static void removeMetrics(ExMetrics metrics) {
+        Metrics.defaultRegistry().removeMetric(metrics.name);
+    }
+
+    private static boolean matchRules(String type, String tag) {
+        assert !Strings.isNullOrEmpty(type);
+        assert !Strings.isNullOrEmpty(tag);
+        String key = type + "." + tag;
+        Rule rule = metricsRules.computeIfAbsent(key, (k) -> new Rule(false, true));
+        Rule.E status = rule.check();
+        if (status != Rule.E.Continue) {
+            return status == Rule.E.Allow;
+        }
+
+        String matchKey = type + ".*";
+        Rule parentRule = metricsRules.computeIfAbsent(matchKey, (k) -> new Rule(false, true));
+        status = parentRule.check();
+        if (status != Rule.E.Continue) {
+            boolean result = status == Rule.E.Allow;
+            rule.setAllow(result);
+            return result;
+        }
+
+        Rule rootRule = metricsRules.computeIfAbsent("*", (k) -> new Rule(false, false));
+        status = rootRule.check();
+        boolean result = status == Rule.E.Allow;
+        rule.setAllow(result);
+        parentRule.setAllow(result);
+        return result;
+    }
+
+    static class Rule {
+        enum E {
+            Allow,
+            Deny,
+            Continue
+        }
+        private long timestamp;
+        private final boolean shadow;
+        private boolean allow;
+
+        public Rule(boolean allow, boolean shadow) {
+            this.timestamp = 0;
+            this.allow = allow;
+            this.shadow = shadow;
+        }
+
+        E check() {
+            E result;
+            if (shadow && System.currentTimeMillis() - timestamp > 100000) {
+                result = E.Continue;
+            } else {
+                result = allow ? E.Allow : E.Deny;
+            }
+            return result;
+        }
+
+        public void setAllow(boolean allow) {
+            this.allow = allow;
+            timestamp = System.currentTimeMillis();
+        }
+    }
+}
+
+

core/src/main/java/com/didichuxing/datachannel/kafka/metrics/TopicRequestMetrics.java

@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.metrics;
+
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+import kafka.network.RequestMetrics;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+public class TopicRequestMetrics {
+
+    private static final String group = "kafka.server";
+    private static final String type = "TopicRequestMetrics";
+
+    private static ConcurrentMap<String, TopicRequestMetrics> allProduceMetrics = new ConcurrentHashMap<>();
+    private static ConcurrentMap<String, TopicRequestMetrics> allFetchMetrics = new ConcurrentHashMap<>();
+
+    // request rate
+    private final ConcurrentHashMap<Integer, ExMeter> requestRateInternal = new ConcurrentHashMap<>();
+    // time a request spent in a request queue
+    private final ExHistogram requestQueueTimeHist;
+    // time a request takes to be processed at the local broker
+    private final ExHistogram localTimeHist;
+    // time a request takes to wait on remote brokers (currently only relevant to fetch and produce requests)
+    private final ExHistogram remoteTimeHist;
+    // time a request is throttled (only relevant to fetch and produce requests)
+    private final ExHistogram throttleTimeHist;
+    // time a response spent in a response queue
+    private final ExHistogram responseQueueTimeHist;
+    // time to send the response to the requester
+    private final ExHistogram responseSendTimeHist;
+    // time to send the response to the requester
+    private final ExHistogram totalTimeHist;
+    // request size in bytes
+    private final ExHistogram requestBytesHist;
+    // response size in bytes
+    private final ExHistogram responseBytesHist;
+    // time for message conversions (only relevant to fetch and produce requests)
+    private final ExHistogram messageConversionsTimeHist;
+    // Temporary memory allocated for processing request (only populated for fetch and produce requests)
+    // This shows the memory allocated for compression/conversions excluding the actual request size
+    private final ExHistogram tempMemoryBytesHist;
+
+    private final ConcurrentHashMap<String, ExMeter> compressionMeters = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<Errors, ExMeter> errorMeters = new ConcurrentHashMap<>();
+
+    private final String topic;
+    private final String requestType;
+
+    public TopicRequestMetrics(String topic, String requestType) {
+        this.topic = topic;
+        this.requestType = requestType;
+
+        Map<String, String> metricsTags = new LinkedHashMap<>();
+        metricsTags.put("request", requestType);
+        metricsTags.put("topic", topic);
+
+        MetricName metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.RequestQueueTimeMs(), metricsTags);
+        requestQueueTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.LocalTimeMs(), metricsTags);
+        localTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.RemoteTimeMs(), metricsTags);
+        remoteTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.ThrottleTimeMs(), metricsTags);
+        throttleTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.ResponseQueueTimeMs(), metricsTags);
+        responseQueueTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.ResponseSendTimeMs(), metricsTags);
+        responseSendTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.TotalTimeMs(), metricsTags);
+        totalTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.RequestBytes(), metricsTags);
+        requestBytesHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.ResponseBytes(), metricsTags);
+        responseBytesHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.MessageConversionsTimeMs(), metricsTags);
+        messageConversionsTimeHist = KafkaExMetrics.createHistogram(metricName, true);
+
+        metricName = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.TemporaryMemoryBytes(), metricsTags);
+        tempMemoryBytesHist = KafkaExMetrics.createHistogram(metricName, true);
+    }
+
+    public void close() {
+        requestRateInternal.forEach((k, v) -> {
+            KafkaExMetrics.removeMetrics(v);
+        });
+        requestRateInternal.clear();
+        KafkaExMetrics.removeMetrics(requestQueueTimeHist);
+        KafkaExMetrics.removeMetrics(localTimeHist);
+        KafkaExMetrics.removeMetrics(remoteTimeHist);
+        KafkaExMetrics.removeMetrics(throttleTimeHist);
+        KafkaExMetrics.removeMetrics(responseQueueTimeHist);
+        KafkaExMetrics.removeMetrics(responseSendTimeHist);
+        KafkaExMetrics.removeMetrics(totalTimeHist);
+        KafkaExMetrics.removeMetrics(requestBytesHist);
+        KafkaExMetrics.removeMetrics(responseBytesHist);
+        KafkaExMetrics.removeMetrics(messageConversionsTimeHist);
+        KafkaExMetrics.removeMetrics(tempMemoryBytesHist);
+        compressionMeters.forEach((k, v) -> {
+            KafkaExMetrics.removeMetrics(v);
+        });
+        compressionMeters.clear();
+        errorMeters.forEach((k, v) -> {
+            KafkaExMetrics.removeMetrics(v);
+        });
+        errorMeters.clear();
+    }
+
+    public ExMeter requestRate(int version) {
+        return requestRateInternal.computeIfAbsent(version, (k) -> {
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("request", requestType);
+            metricsTags.put("topic", topic);
+            metricsTags.put("version", String.valueOf(version));
+            MetricName name = KafkaExMetrics.createMetricsName(group, type, RequestMetrics.RequestsPerSec(), metricsTags);
+            return KafkaExMetrics.createMeter(name, "requests", TimeUnit.SECONDS);
+        });
+    }
+
+    ExMeter compressionMetrics(String compression) {
+        return compressionMeters.computeIfAbsent(compression, (k) -> {
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("request", requestType);
+            metricsTags.put("topic", topic);
+            metricsTags.put("compression", compression);
+            MetricName metricName = KafkaExMetrics.createMetricsName(group, type, "CompressionPerSec", metricsTags);
+            return KafkaExMetrics.createMeter(metricName, "record compression", TimeUnit.SECONDS);
+        });
+    }
+
+    ExMeter errorMetrics(Errors error, String name) {
+        return errorMeters.computeIfAbsent(error, (k) -> {
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("request", requestType);
+            metricsTags.put("topic", topic);
+            metricsTags.put("error", name);
+            MetricName metricName = KafkaExMetrics.createMetricsName(group, type, "ErrorPerSec", metricsTags);
+            return KafkaExMetrics.createMeter(metricName, "errors", TimeUnit.SECONDS);
+        });
+    }
+
+    public void markError(Errors errors, String name) {
+        errorMetrics(errors, name).meter().mark(1);
+    }
+
+    public void markRequest(int verrsion) {
+        requestRate(verrsion).meter().mark();
+        AppIdTopicMetrics appIdTopicMetrics = KafkaExMetrics.appIdTopicMetrics(topic);
+        if (null != appIdTopicMetrics) {
+            if (requestType.equals(ApiKeys.PRODUCE.name)) {
+                appIdTopicMetrics.markProduceApiVersion(verrsion);
+            } else {
+                appIdTopicMetrics.markFetchApiVersion(verrsion);
+            }
+        }
+        AppIdHostTopicMetrics appIdHostTopicMetrics = KafkaExMetrics.appIdHostTopicMetrics(topic);
+        if (null != appIdHostTopicMetrics) {
+            appIdHostTopicMetrics.netRequestInRate().mark();
+        }
+    }
+
+
+    public void markCompress(String compression) {
+        compressionMetrics(compression).meter().mark();
+        AppIdTopicMetrics appIdTopicMetrics = KafkaExMetrics.appIdTopicMetrics(topic);
+        if (null != appIdTopicMetrics) {
+            appIdTopicMetrics.markRecordCompression(compression);
+        }
+    }
+
+    public static TopicRequestMetrics getProduceRequestMetrics(String topic) {
+        return allProduceMetrics.computeIfAbsent(topic, (k) -> new TopicRequestMetrics(topic, ApiKeys.PRODUCE.name));
+    }
+
+    public static TopicRequestMetrics getFetchRequestMetrics(String topic) {
+        return allFetchMetrics.computeIfAbsent(topic, (k) -> new TopicRequestMetrics(topic, ApiKeys.FETCH.name));
+    }
+
+    public static void removeRequestMetrics(String topic) {
+        TopicRequestMetrics metrics = allProduceMetrics.remove(topic);
+        if (metrics != null) {
+            metrics.close();
+        }
+
+        metrics = allFetchMetrics.remove(topic);
+        if (metrics != null) {
+            metrics.close();
+        }
+    }
+
+    public Histogram requestQueueTimeHist() {
+        return requestQueueTimeHist.histogram();
+    }
+
+    public Histogram localTimeHist() {
+        return localTimeHist.histogram();
+    }
+
+    public Histogram remoteTimeHist() {
+        return remoteTimeHist.histogram();
+    }
+
+    public Histogram throttleTimeHist() {
+        return throttleTimeHist.histogram();
+    }
+
+    public Histogram responseQueueTimeHist() {
+        return responseQueueTimeHist.histogram();
+    }
+
+    public Histogram responseSendTimeHist() {
+        return responseSendTimeHist.histogram();
+    }
+
+    public Histogram totalTimeHist() {
+        return totalTimeHist.histogram();
+    }
+
+    public Histogram requestBytesHist() {
+        return requestBytesHist.histogram();
+    }
+
+    public Histogram responseBytesHist() {
+        return responseBytesHist.histogram();
+    }
+
+    public Histogram messageConversionsTimeHist() {
+        return messageConversionsTimeHist.histogram();
+    }
+
+    public Histogram tempMemoryBytesHist() {
+        return tempMemoryBytesHist.histogram();
+    }
+
+    static public String getType() {
+        return type;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/report/SessionReport.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.report;
+
+import com.didichuxing.datachannel.kafka.security.authorizer.SessionManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class SessionReport {
+
+    private static final Logger log = LoggerFactory.getLogger(SessionReport.class);
+    private String topicHeartBeatUrlPrefix;
+
+    private SessionReport() {
+    }
+
+    static public SessionReport getInstance() {
+        return SessionReportHolder.INSTANCE;
+    }
+
+    public void start(String clusterId, int brokerId, String urlPrefix, int sessionReportTimes, ScheduledExecutorService scheduledExecutorService) {
+        topicHeartBeatUrlPrefix = urlPrefix;
+        log.info("Session reporter startup");
+        scheduledExecutorService.scheduleWithFixedDelay(() -> {
+            try {
+                sendTopicHeartBeat(clusterId, brokerId);
+            } catch (Throwable t) {
+                log.error("Uncaught error session report I/O thread: ", t);
+            }
+        }, sessionReportTimes, sessionReportTimes, TimeUnit.MILLISECONDS);
+    }
+
+    public String getTopicHeartBeat() {
+        Map<String, List<String>> topicProduceUser = new HashMap<>();
+        Map<String, List<String>> topicFetchUser = new HashMap<>();
+        SessionManager.getInstance().getSessions().forEach((userName, v1) ->
+                v1.forEach((sessionKey, session) ->
+                        session.getAccessTopics().forEach((topicName, v3) -> {
+                            if (v3.length == 2 && !"unknown".equals(session.getClientId())) {
+                                if (v3[0] != null) {
+                                    topicFetchUser.putIfAbsent(topicName, new ArrayList<>());
+                                    topicFetchUser.get(topicName).add(getIdKey(userName, session.getHostAddress() + ":" + session.getClientPort(), session.getClientVersion(), session.getClientId(), v3[0].getActiveTimestamp()));
+                                }
+                                if (v3[1] != null) {
+                                    topicProduceUser.putIfAbsent(topicName, new ArrayList<>());
+                                    topicProduceUser.get(topicName).add(getIdKey(userName, session.getHostAddress() + ":" + session.getClientPort(), session.getClientVersion(), session.getClientId(), v3[1].getActiveTimestamp()));
+                                }
+                            }
+                        })));
+        Map<String, Map<String, List<String>>> resultMap = new HashMap<>();
+        resultMap.put("produce", topicProduceUser);
+        resultMap.put("fetch", topicFetchUser);
+        // String result = JSON.toJSONString(resultMap);
+        log.debug("Session report: {}", resultMap);
+        return "";
+    }
+
+    public void sendTopicHeartBeat(String clusterId, int brokerId) {
+        Map<String, String> paramMap = new HashMap<>();
+        paramMap.put("clusterId", clusterId);
+        paramMap.put("brokerId", String.valueOf(brokerId));
+        String data = getTopicHeartBeat();
+
+    }
+
+    public void shutdown() {
+    }
+
+    public String getIdKey(String appId, String ip, String version, String clientId, long activeTimestamp) {
+        return appId + "#" + ip + "#" + version + "#" + clientId + "#" + activeTimestamp;
+    }
+
+    private static class SessionReportHolder {
+        private static final SessionReport INSTANCE = new SessionReport();
+
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/authorizer/AccessStatus.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.authorizer;
+
+public enum AccessStatus {
+    Deny,
+    Continue,
+    Allow;
+
+    static AccessStatus from(String value) {
+        if (value.equals(Deny.name())) {
+            return Deny;
+        } else if (value.equals(Allow.name())) {
+            return Allow;
+        } else {
+            return Continue;
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/authorizer/AccessStatusAndTimestamp.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.authorizer;
+
+public class AccessStatusAndTimestamp {
+
+    //access status for topic and operation
+    final private AccessStatus accessStatus;
+    //acl cache timestamp .if not match cache has expired
+    volatile private long aclTimestamp;
+    //topic and operation active time.
+    volatile private long activeTimestamp;
+
+    public AccessStatusAndTimestamp(AccessStatus accessStatus, long aclTimestamp) {
+        this.accessStatus = accessStatus;
+        this.aclTimestamp = aclTimestamp;
+        activeTimestamp = System.currentTimeMillis();
+    }
+
+    public AccessStatus getAccessStatus() {
+        return accessStatus;
+    }
+
+    public long getAclTimestamp() {
+        return aclTimestamp;
+    }
+
+    public long getActiveTimestamp() {
+        return activeTimestamp;
+    }
+
+    public void setActiveTimestamp(long activeTimestamp) {
+        this.activeTimestamp = activeTimestamp;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/authorizer/Operation.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.authorizer;
+
+public enum Operation {
+    Read,
+    Write,
+    Create,
+    Delete,
+    Describe,
+    Alert,
+    ClusterAction,
+    DescribeConfigs,
+    AlterConfigs,
+    IdempotentWrite,
+    Other;
+
+    static Operation from(kafka.security.auth.Operation operation) {
+        return from(operation.name());
+    }
+
+    static Operation from(String value) {
+        if (value.equals(Read.name())) {
+            return Read;
+        } else if (value.equals(Write.name())) {
+            return Write;
+        } else if (value.equals(Create.name())) {
+            return Create;
+        } else if (value.equals(Delete.name())) {
+            return Delete;
+        } else if (value.equals(Describe.name())) {
+            return Describe;
+        } else if (value.equals(Alert.name())) {
+            return Alert;
+        } else if (value.equals(ClusterAction.name())) {
+            return ClusterAction;
+        } else if (value.equals(DescribeConfigs.name())) {
+            return DescribeConfigs;
+        } else if (value.equals(AlterConfigs.name())) {
+            return AlterConfigs;
+        } else if (value.equals(IdempotentWrite.name())) {
+            return IdempotentWrite;
+        } else {
+            return Other;
+        }
+    }
+
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/authorizer/Session.java

@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.authorizer;
+
+import com.didichuxing.datachannel.kafka.security.login.LoginManager;
+import com.didichuxing.datachannel.kafka.security.login.User;
+import com.didichuxing.datachannel.kafka.util.KafkaUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Session class use to manager a user client access the server.
+ * it use to cache topic access status.
+ */
+public class Session {
+
+    final static private int SESSION_EXPIRD_TIMEMS = 6 * 60 * 1000;
+    final static private int ACCESS_STATUS_STORE_SIZE = 2;
+
+    private static final Logger log = LoggerFactory.getLogger("kafka.authorizer.logger");
+
+    public static final Session AdminSession = new Session(LoginManager.admin_USER, "localhost", -1);
+    public static final Session UnkonwnSession = new Session(LoginManager.unknown_USER, "localhost", -1);
+
+    //session user
+    final private User user;
+    //session clinet hostaddress
+    final private String hostAddress;
+    final private int clientPort;
+    //session active time
+    volatile private long activeTimestamp;
+    // client version   0.8/0.9 0, 0.10.0 1, 0.10.1/0.10.2 2
+    private String clientVersion = "unknown";
+    // client id
+    private String clientId = "unknown";
+    //map topic name to AccessStatusAndTimestamp. the value value is array. index 0 is read. index 1 is write.
+    private ConcurrentHashMap<String, AccessStatusAndTimestamp[]> accessTopics = new ConcurrentHashMap<>();
+
+    public Session(User user, String hostAddress) {
+        this(user, hostAddress, -1);
+    }
+
+    public Session(User user, String hostAddress, int clientPort) {
+        this.user = user;
+        this.hostAddress = hostAddress;
+        this.clientPort = clientPort;
+        this.activeTimestamp = System.currentTimeMillis();
+    }
+
+    public User getUser() {
+        return user;
+    }
+
+    public ConcurrentHashMap<String, AccessStatusAndTimestamp[]> getAccessTopics() {
+        return accessTopics;
+    }
+
+    public void setActiveTimestamp(long activeTimestamp) {
+        this.activeTimestamp = activeTimestamp;
+    }
+
+    public long getActiveTimestamp() {
+        return activeTimestamp;
+    }
+
+    //Get access check status in session
+    public AccessStatusAndTimestamp getAccessStatus(String topicname, Operation operation) {
+        AccessStatusAndTimestamp[] accessTopic = accessTopics.get(topicname);
+        if (accessTopic != null) {
+            AccessStatusAndTimestamp accessStatusAndTimestamp = accessTopics.get(topicname)[operation.ordinal()];
+            if (accessStatusAndTimestamp == null) {
+                return null;
+            }
+
+            //update active time
+            accessStatusAndTimestamp.setActiveTimestamp(System.currentTimeMillis());
+            return accessStatusAndTimestamp;
+        } else {
+            return null;
+        }
+    }
+
+    //Cache access check status in session
+    public void setAccessStatus(String topicName, Operation operation,
+                                AccessStatusAndTimestamp accessStatusAndTimestamp) {
+        if (operation.ordinal() > Operation.Write.ordinal()) {
+            return;
+        }
+        AccessStatusAndTimestamp[] status =  accessTopics.get(topicName);
+        if (status == null) {
+            //ACCESS_STATUS_STORE_SIZE is 2. AccessStatusAndTimestamp store access status. 0 use for read.
+            // 1 use for write.
+            status = new AccessStatusAndTimestamp[ACCESS_STATUS_STORE_SIZE];
+            status[operation.ordinal()] = accessStatusAndTimestamp;
+            accessTopics.putIfAbsent(topicName, status);
+        } else {
+            status[operation.ordinal()] = accessStatusAndTimestamp;
+        }
+    }
+
+    public boolean checkExpierd() {
+        //check activetime
+        if (System.currentTimeMillis() - activeTimestamp > SESSION_EXPIRD_TIMEMS) {
+            return true;
+        }
+
+        //check all the topics. if it's expire set access status is null
+        Iterator<Map.Entry<String, AccessStatusAndTimestamp[]>> iterator = accessTopics.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, AccessStatusAndTimestamp[]> entry = iterator.next();
+            AccessStatusAndTimestamp[] status = entry.getValue();
+            boolean allNull = true;
+            //topic read or write expired
+            for (int i = 0; i < ACCESS_STATUS_STORE_SIZE; i++) {
+                if (status[i] != null) {
+                    if (System.currentTimeMillis() - status[i].getActiveTimestamp() > SESSION_EXPIRD_TIMEMS) {
+                        log.debug("Session manager delete session User = {}, Host = {}, Topic = {} Operation = {}",
+                                user.getUsername(), hostAddress, entry.getKey(), i == 0 ? "Read" : "Write");
+                        status[i] = null;
+                    } else {
+                        allNull = false;
+                    }
+                }
+            }
+
+            //topic is expired
+            if (allNull) {
+                log.debug("Session manager delete session User = {}, Host = {}, Topic = {}", user.getUsername(),
+                        hostAddress, entry.getKey());
+                iterator.remove();
+            }
+        }
+        return false;
+    }
+
+    public void setClientVersionByApi(Short apiKey, Short apiVersion) {
+        this.clientVersion = KafkaUtils.apiVersionToKafkaVersion(apiKey, apiVersion);
+    }
+
+    public void setClientVersion(String clientVersion) {
+        this.clientVersion = clientVersion;
+    }
+
+    public String getClientVersion() {
+        return clientVersion;
+    }
+
+    public void setClientId(String clientId) {
+        this.clientId = clientId;
+    }
+
+    public String getClientId() {
+        return clientId;
+    }
+
+    public String getHostAddress() {
+        return hostAddress;
+    }
+
+    public int getClientPort() {
+        return clientPort;
+    }
+
+    public String getUsername() {
+        return user.getUsername();
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/authorizer/SessionManager.java

@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.authorizer;
+
+import com.didichuxing.datachannel.kafka.security.login.LoginManager;
+import com.didichuxing.datachannel.kafka.security.login.User;
+import kafka.network.RequestChannel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *  This class use by mananger sessions.
+ *  Each appId and ip address have one session
+ *  There are two layer index for manger session. the first layer index by aapid,
+ *  the second layer index by address
+ *  it start a schedule task to check and remove expired sessions
+ */
+public class SessionManager {
+
+    final static private int CHECK_SESSION_TIMEMS = 60 * 1000;
+    final static private int PRINT_SESSION_TIMEMS = 15 * 60 * 1000;
+
+    private static final Logger log = LoggerFactory.getLogger("kafka.authorizer.logger");
+
+    // map username to submap that map hostname to seesion.
+    final private ConcurrentHashMap<String, ConcurrentHashMap<String, Session>> sessions = new ConcurrentHashMap<>();
+
+    private ScheduledFuture<?> scheduledFuture;
+    private int maxSessionsPerUser;
+
+    private SessionManager() {
+    }
+
+    static public SessionManager getInstance() {
+        return SessionMangerHolder.INSTANCE;
+    }
+
+    public void start(ScheduledExecutorService scheduledExecutorService) {
+        this.start(scheduledExecutorService, 2000);
+    }
+
+    public void start(ScheduledExecutorService scheduledExecutorService, int maxSessionsPerUser) {
+        log.info("Session manager startup");
+        this.maxSessionsPerUser = maxSessionsPerUser;
+        scheduledFuture = scheduledExecutorService.scheduleWithFixedDelay(() -> {
+            try {
+                cleanSession();
+            } catch (Throwable throwable) {
+                log.error("Session manager exception: ", throwable);
+            }
+        }, CHECK_SESSION_TIMEMS, CHECK_SESSION_TIMEMS, TimeUnit.MILLISECONDS);
+
+        scheduledFuture = scheduledExecutorService.scheduleWithFixedDelay(() -> {
+            try {
+                printSession();
+            } catch (Throwable throwable) {
+                log.error("Session manager exception: ", throwable);
+            }
+        }, PRINT_SESSION_TIMEMS, PRINT_SESSION_TIMEMS, TimeUnit.MILLISECONDS);
+    }
+
+    public void shutdown() {
+        log.info("Session manager shutdown");
+        if (scheduledFuture != null) {
+            scheduledFuture.cancel(true);
+        }
+        sessions.clear();
+    }
+
+    private void cleanSession() {
+        log.trace("Session manager clean session");
+        Iterator<Map.Entry<String, ConcurrentHashMap<String, Session>>> mapIterator = sessions.entrySet().iterator();
+        while (mapIterator.hasNext()) {
+            Map.Entry<String, ConcurrentHashMap<String, Session>> mapEntry = mapIterator.next();
+            ConcurrentHashMap<String, Session> map = mapEntry.getValue();
+
+            // clean session
+            int out = map.size() - this.maxSessionsPerUser;
+            if (out > 0) {
+                log.info("The number of user {} sessions {} has exceeded the maximum value of {}, will be clean.", mapEntry.getKey(), map.size(), this.maxSessionsPerUser);
+                Comparator<Session> comparator =
+                        Comparator.<Session>comparingDouble(Session::getActiveTimestamp)
+                                .thenComparingInt(Session::hashCode);
+                TreeSet<Session> sortSet = new TreeSet<>(comparator);
+                sortSet.addAll(map.values());
+                for (Session oldleast : sortSet) {
+                    map.remove(getSessionKey(oldleast));
+                    log.info("Session manager clean out of max sessions for User = {}, Host = {}, Port = {}",
+                            oldleast.getUsername(), oldleast.getHostAddress(), oldleast.getClientPort());
+                    out --;
+                    if (out <= 0) {
+                        break;
+                    }
+                }
+            }
+
+            Iterator<Map.Entry<String, Session>> sessionIterator = map.entrySet().iterator();
+            while (sessionIterator.hasNext()) {
+                Map.Entry<String, Session> sessionEntry = sessionIterator.next();
+                Session session = sessionEntry.getValue();
+                if (session.checkExpierd()) {
+                    sessionIterator.remove();
+                    log.info("Session manager clean session User = {}, Host = {}, Port = {}",
+                            session.getUsername(), session.getHostAddress(), session.getClientPort());
+                }
+            }
+
+            if (map.isEmpty()) {
+                mapIterator.remove();
+            }
+        }
+    }
+
+    private void printSession() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("Sessions: [");
+        sessions.forEach((userName, sessions) -> {
+            sb.append(String.format("[userName: %s, hosts: [", userName));
+            sessions.forEach((host, session) -> {
+                sb.append(String.format("[host: %s, version: %s, clientid: %s, topics: [",
+                        host, session.getClientVersion(), session.getClientId()));
+                ConcurrentHashMap<String, AccessStatusAndTimestamp[]> accessTopics = session.getAccessTopics();
+                accessTopics.forEach((topic, statusAndTimestamps) -> {
+                    String operation = "produce+fetch";
+                    if (statusAndTimestamps[0] == null) {
+                        operation = "produce";
+                    } else if (statusAndTimestamps[1] == null) {
+                        operation = "consume";
+                    }
+                    sb.append(String.format("[topic: %s, operation: %s]", topic, operation));
+                });
+                if (sb.charAt(sb.length() - 1) == ',') {
+                    sb.deleteCharAt(sb.length() - 1);
+                }
+                sb.append("],");
+            });
+            if (sb.charAt(sb.length() - 1) == ',') {
+                sb.deleteCharAt(sb.length() - 1);
+            }
+            sb.append("],");
+        });
+        if (sb.charAt(sb.length() - 1) == ',') {
+            sb.deleteCharAt(sb.length() - 1);
+        }
+        sb.append("]");
+        log.info(sb.toString());
+    }
+
+    private String getSessionKey(Session session) {
+        return session.getHostAddress() + ":" + session.getClientPort();
+    }
+
+    public Session getSession(RequestChannel.Session kafkaSession) {
+        String userName = kafkaSession.principal().getName();
+        String hostAddress = kafkaSession.clientAddress().getHostAddress();
+        String sessionKey = hostAddress + ":" + kafkaSession.clientPort();
+        ConcurrentHashMap<String, Session> map = sessions.get(userName);
+        if (map == null) {
+            map = new ConcurrentHashMap<>();
+            sessions.putIfAbsent(userName, map);
+        }
+
+        Session session = map.get(sessionKey);
+        if (session == null) {
+//            User user = LoginManager.getInstance().getUser(userName);
+//            if (user == null) {
+//                //log.warn("Session manager can't fount the user User = {}, Host = {}, generate tmp session", userName, hostAddress);
+//                session = new Session(new User(userName, "", false), hostAddress, kafkaSession.clientPort());
+//                return session;
+//            }
+            User user = new User(userName, "", false);
+            session = new Session(user, hostAddress, kafkaSession.clientPort());
+            log.info("Session manager create session User = {}, Host = {}, Port = {}", userName, hostAddress, kafkaSession.clientPort());
+            map.putIfAbsent(sessionKey, session);
+        }
+
+        session.setActiveTimestamp(System.currentTimeMillis());
+        return session;
+    }
+
+    public ConcurrentHashMap<String, ConcurrentHashMap<String, Session>> getSessions() {
+        return sessions;
+    }
+
+    private static class SessionMangerHolder {
+
+        private static final SessionManager INSTANCE = new SessionManager();
+
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/login/LoginManager.java

@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.login;
+
+import kafka.cluster.Broker;
+import kafka.cluster.EndPoint;
+import kafka.zk.KafkaZkClient;
+import org.apache.kafka.common.network.ListenerName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.collection.JavaConverters;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+
+public class LoginManager {
+
+    final private static String USER_ANONYMOUS = "ANONYMOUS";
+    final private static String USER_admin = "admin";
+
+    final public static User ANONYMOUS_USER = new User(USER_ANONYMOUS, "*", true);
+    final public static User admin_USER = new User(USER_admin, "*", true);
+    final public static User unknown_USER = new User("unknown", "*", true);
+
+    final private static ListenerName DEFAULT_LISTENER_NAME = new ListenerName("SASL_PLAINTEXT");
+
+    private static final Logger log = LoggerFactory.getLogger("kafka.authorizer.logger");
+    private static final Logger userErrorlog = LoggerFactory.getLogger("userError");
+
+    final private String CACHE_NAME = "kafka_login";
+
+    private KafkaZkClient zkUtil;
+
+    private ListenerName listenerName;
+
+    private LoginManager(){}
+
+    static public LoginManager getInstance() {
+        return LoginMangerHolder.INSTANCE;
+    }
+
+    public void start(String clusterId, int brokerId, KafkaZkClient zkUtil,
+                      ScheduledExecutorService scheduledExecutorService,
+                      String gatewayUrl, List<String> defaultUsers) {
+        this.start(clusterId, brokerId, DEFAULT_LISTENER_NAME, zkUtil, scheduledExecutorService, gatewayUrl, defaultUsers);
+    }
+
+    public void start(String clusterId, int brokerId, ListenerName listenerName, KafkaZkClient zkUtil,
+                      ScheduledExecutorService scheduledExecutorService,
+                      String gatewayUrl, List<String> defaultUsers) {
+        log.info("Login Manager startup");
+        this.listenerName = listenerName;
+        this.zkUtil = zkUtil;
+
+        List<String> systemUsers = new ArrayList<>();
+        systemUsers.add(String.format("%s:%s:%s",
+                ANONYMOUS_USER.getUsername(), ANONYMOUS_USER.getPassword(), admin_USER.isSuperUser()));
+        systemUsers.add(String.format("%s:%s:%s",
+                admin_USER.getUsername(), admin_USER.getPassword(), admin_USER.isSuperUser()));
+        if (defaultUsers != null)
+            systemUsers.addAll(defaultUsers);
+    }
+
+    public void shutdown() {
+        log.info("Login Manager shutdown");
+    }
+
+    /**
+     * check login
+     * @param username
+     * @param password
+     * @return login status
+     */
+    public boolean login(String username, String password, String host) {
+        User user = getUser(username);
+        if (user != null) {
+            if (username.equals(USER_ANONYMOUS)) {
+                userErrorlog.error("User = {} from {} login failed. no permission for this user.",
+                        username, host);
+                return false;
+            }
+
+            if (user.getUsername().equals(USER_admin)) {
+                List<Broker> brokers = JavaConverters.seqAsJavaList(zkUtil.getAllBrokersInCluster().seq());
+                for (Broker broker : brokers) {
+                    EndPoint endPoint = broker.endPoint(this.listenerName);
+                    String brokerHost = "";
+                    try {
+                        brokerHost = InetAddress.getByName(endPoint.host()).getHostAddress();
+                    } catch (Exception e) {
+                        log.error("User = {} from {} login failed", username, host, e);
+                    }
+                    if (host.equals(brokerHost) || host.equals("127.0.0.1")) {
+                        log.info("User = {} from {} login success", username, host);
+                        return true;
+                    }
+                }
+                userErrorlog.error("User = {} from {} login failed. no permission for this user.",
+                        username, host);
+                return false;
+            }
+
+            boolean success = false;
+            try {
+                success = password.equals(SecurityUtils.decrypt(user.getPassword()));
+            } catch (RuntimeException e) {
+                log.warn("User = {} from {} decrypt password failed", username, host);
+            }
+
+            if (password.equals(user.getPassword()) || success) {
+                log.info("User = {} from {} login success", username, host);
+                return true;
+            }
+        }
+        log.error("User = {} from {} login failed", username, host);
+        userErrorlog.error("User = {} and Password = {} from {} login failed. error username or password.",
+                username, hiddenText(password), host);
+        return false;
+    }
+
+    /**
+     * get Kafka User
+     * @param userName
+     * @return user
+     */
+    public User getUser(String userName) {
+        return null;
+    }
+
+    private String hiddenText(String s) {
+        if (s == null || s.isEmpty()) return s;
+        StringBuilder builder = new StringBuilder();
+        builder.append(s.charAt(0));
+        for (int i = 1; i < s.length() - 1; i++) {
+            builder.append('*');
+        }
+        builder.append(s.charAt(s.length() - 1));
+        return builder.toString();
+    }
+
+    private static class LoginMangerHolder{
+
+        private static final LoginManager INSTANCE = new LoginManager();
+
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/login/SecurityUtils.java

@@ -0,0 +1,61 @@
+package com.didichuxing.datachannel.kafka.security.login;
+
+import kafka.server.KafkaConfig;
+import org.apache.kafka.common.config.types.Password;
+import scala.Option;
+
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.DESKeySpec;
+import java.security.SecureRandom;
+import java.util.Arrays;
+import java.util.Base64;
+
+public class SecurityUtils {
+    private static String ENCRYPT_KEY = "123456";
+    private static String ENCRYPT_ALGORITHM = "DES";
+    private static boolean isSec = false;
+
+    public static void start(KafkaConfig config) {
+        // ALGORITHM = config.passwordEncoderCipherAlgorithm();
+        Option<String> secret = config.passwordEncoderSecret().map(Password::value);
+        if (secret.isDefined()) {
+            isSec = true;
+            ENCRYPT_KEY = secret.get();
+        }
+    }
+
+    public static String encrypt(String data) {
+        return Base64.getEncoder().encodeToString(des(data.getBytes(), Cipher.ENCRYPT_MODE));
+    }
+
+    public static String decrypt(String data) {
+        if (!isSec) {
+            return data;
+        }
+        return new String(des(Base64.getDecoder().decode(data), Cipher.DECRYPT_MODE));
+    }
+
+    private static byte[] des(byte[] data, int mode) {
+        try {
+            byte[] key = Arrays.copyOf(ENCRYPT_KEY.getBytes(), 8);
+            DESKeySpec desKeySpec = new DESKeySpec(key);
+
+            SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(ENCRYPT_ALGORITHM);
+            SecretKey secretKey = keyFactory.generateSecret(desKeySpec);
+
+            Cipher cipher = Cipher.getInstance(ENCRYPT_ALGORITHM);
+            cipher.init(mode, secretKey, new SecureRandom());
+            return cipher.doFinal(data);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static void main(String[] args) {
+        String p = encrypt("1qaz2wsx");
+        System.out.println(p);
+        System.out.println(decrypt(p));
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/login/User.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.login;
+
+public class User {
+
+    private String username;
+    private String password;
+    private boolean superUser;
+
+    public User() {
+    }
+
+    public User(String username, String password, boolean superUser) {
+        this.username = username;
+        this.password = password;
+        this.superUser = superUser;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public boolean isSuperUser() {
+        return superUser;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        User user = (User) obj;
+        return username.equals(user.username) &&
+                password.equals(user.password) &&
+                superUser == user.superUser;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("username: %s, password: %s, superuser: %b", username, password, superUser);
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/sasl/plain/PlainLoginModule.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.sasl.plain;
+
+import javax.security.auth.Subject;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.login.LoginException;
+import javax.security.auth.spi.LoginModule;
+import java.util.Map;
+
+public class PlainLoginModule implements LoginModule {
+
+    private static final String USERNAME_CONFIG = "username";
+    private static final String PASSWORD_CONFIG = "password";
+
+    static {
+        PlainSaslServerProvider.initialize();
+    }
+
+    @Override
+    public void initialize(Subject subject, CallbackHandler callbackHandler, Map<String, ?> sharedState, Map<String, ?> options) {
+        String username = (String) options.get(USERNAME_CONFIG);
+        if (username != null)
+            subject.getPublicCredentials().add(username);
+        String password = (String) options.get(PASSWORD_CONFIG);
+        if (password != null)
+            subject.getPrivateCredentials().add(password);
+    }
+
+    @Override
+    public boolean login() throws LoginException {
+        return true;
+    }
+
+    @Override
+    public boolean logout() throws LoginException {
+        return true;
+    }
+
+    @Override
+    public boolean commit() throws LoginException {
+        return true;
+    }
+
+    @Override
+    public boolean abort() throws LoginException {
+        return false;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/sasl/plain/PlainSaslServer.java

@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.sasl.plain;
+
+import com.didichuxing.datachannel.kafka.security.login.LoginManager;
+import org.apache.kafka.common.errors.SaslAuthenticationException;
+
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
+import javax.security.sasl.SaslServerFactory;
+import java.io.UnsupportedEncodingException;
+import java.nio.channels.SocketChannel;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Simple SaslServer implementation for SASL/PLAIN. In order to make this implementation
+ * fully pluggable, authentication of username/password is fully contained within the
+ * server implementation.
+ * <p>
+ * Valid users with passwords are specified in the Jaas configuration file. Each user
+ * is specified with user_<username> as key and <password> as value. This is consistent
+ * with Zookeeper Digest-MD5 implementation.
+ * <p>
+ * To avoid storing clear passwords on disk or to integrate with external authentication
+ * servers in production systems, this module can be replaced with a different implementation.
+ *
+ */
+public class PlainSaslServer implements SaslServer {
+
+    public static final String PLAIN_MECHANISM = "PLAIN";
+    private static final String JAAS_USER_PREFIX = "user_";
+
+    private boolean complete;
+    private String authorizationID;
+    private Map<String, ?> props;
+
+    public PlainSaslServer(String serverName, Map<String, ?> props) {
+        this.props = props;
+    }
+
+    @Override
+    public byte[] evaluateResponse(byte[] response) throws SaslException {
+        /*
+         * Message format (from https://tools.ietf.org/html/rfc4616):
+         *
+         * message   = [authzid] UTF8NUL authcid UTF8NUL passwd
+         * authcid   = 1*SAFE ; MUST accept up to 255 octets
+         * authzid   = 1*SAFE ; MUST accept up to 255 octets
+         * passwd    = 1*SAFE ; MUST accept up to 255 octets
+         * UTF8NUL   = %x00 ; UTF-8 encoded NUL character
+         *
+         * SAFE      = UTF1 / UTF2 / UTF3 / UTF4
+         *                ;; any UTF-8 encoded Unicode character except NUL
+         */
+
+        String[] tokens;
+        try {
+            tokens = new String(response, "UTF-8").split("\u0000");
+        } catch (UnsupportedEncodingException e) {
+            throw new SaslException("UTF-8 encoding not supported", e);
+        }
+        if (tokens.length != 3)
+            throw new SaslException("Invalid SASL/PLAIN response: expected 3 tokens, got " + tokens.length);
+        authorizationID = tokens[0];
+        String username = tokens[1];
+        String password = tokens[2];
+
+        if (username.isEmpty()) {
+            throw new SaslException(String.format("Authentication failed: username:%s not specified", authorizationID));
+        }
+        if (password.isEmpty()) {
+            throw new SaslException(String.format("Authentication failed: password:%s not specified for username:%s", password, authorizationID));
+        }
+
+        //user name should be cluserId.appId or appId
+        // cluserid.appid used to service discovery.
+        // if programe run at this step. clusterid is not usefull.
+        // it should trim the clusterId.
+        if (authorizationID.isEmpty()) {
+            int index = username.indexOf('.');
+            if (index == -1 || index == username.length()) {
+                authorizationID = username;
+            } else {
+                authorizationID = username.substring(index + 1);
+            }
+        } else {
+            int index = authorizationID.indexOf('.');
+            if (index != -1 && index != authorizationID.length()) {
+                authorizationID = authorizationID.substring(index + 1);
+            }
+        }
+
+        try {
+            boolean login = LoginManager.getInstance().login(
+                    authorizationID, password, props.get("client_ip").toString());
+            if (!login) {
+                throw new SaslAuthenticationException(
+                    String.format("Authentication failed: Invalid username:%s or password:%s",
+                        authorizationID, password));
+            }
+        } catch (Exception e) {
+            throw new SaslAuthenticationException("Authentication failed: exception: ", e);
+        }
+        complete = true;
+        return new byte[0];
+    }
+
+    @Override
+    public String getAuthorizationID() {
+        if (!complete)
+            throw new IllegalStateException("Authentication exchange has not completed");
+        return authorizationID;
+    }
+
+    @Override
+    public String getMechanismName() {
+        return PLAIN_MECHANISM;
+    }
+
+    @Override
+    public Object getNegotiatedProperty(String propName) {
+        if (!complete)
+            throw new IllegalStateException("Authentication exchange has not completed");
+        return null;
+    }
+
+    @Override
+    public boolean isComplete() {
+        return complete;
+    }
+
+    @Override
+    public byte[] unwrap(byte[] incoming, int offset, int len) throws SaslException {
+        if (!complete)
+            throw new IllegalStateException("Authentication exchange has not completed");
+        return Arrays.copyOfRange(incoming, offset, offset + len);
+    }
+
+    @Override
+    public byte[] wrap(byte[] outgoing, int offset, int len) throws SaslException {
+        if (!complete)
+            throw new IllegalStateException("Authentication exchange has not completed");
+        return Arrays.copyOfRange(outgoing, offset, offset + len);
+    }
+
+    @Override
+    public void dispose() throws SaslException {
+    }
+
+    public static class PlainSaslServerFactory implements SaslServerFactory {
+
+        @Override
+        public SaslServer createSaslServer(String mechanism, String protocol, String serverName, Map<String, ?> props, CallbackHandler cbh)
+            throws SaslException {
+            if (!PLAIN_MECHANISM.equals(mechanism)) {
+                throw new SaslException(String.format("Mechanism \'%s\' is not supported. Only PLAIN is supported.", mechanism));
+            }
+            return new PlainSaslServer(serverName, props);
+        }
+
+        @Override
+        public String[] getMechanismNames(Map<String, ?> props) {
+            String noPlainText = (String) props.get(Sasl.POLICY_NOPLAINTEXT);
+            if ("true".equals(noPlainText))
+                return new String[]{};
+            else
+                return new String[]{PLAIN_MECHANISM};
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/security/sasl/plain/PlainSaslServerProvider.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.security.sasl.plain;
+
+import java.security.Provider;
+import java.security.Security;
+
+public class PlainSaslServerProvider extends Provider {
+
+    private static final long serialVersionUID = 1L;
+
+    protected PlainSaslServerProvider() {
+        super("Simple SASL/PLAIN Server Provider", 1.0, "Simple SASL/PLAIN Server Provider for Kafka");
+        super.put("SaslServerFactory." + PlainSaslServer.PLAIN_MECHANISM,
+                PlainSaslServer.PlainSaslServerFactory.class.getName());
+    }
+
+    public static void initialize() {
+        Security.addProvider(new PlainSaslServerProvider());
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/server/DiskLoadProtector.java

@@ -0,0 +1,614 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.server;
+
+
+import com.didichuxing.datachannel.kafka.metrics.ExMeter;
+import com.didichuxing.datachannel.kafka.metrics.KafkaExMetrics;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.MetricName;
+import org.apache.kafka.common.Configurable;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.mina.util.ConcurrentHashSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+//This class used to protect read file from disk. it avoid that disk read by random for too many read request.
+// 1. check whether the read request can invoke physical disk read. if no allow the access otherwise, continue
+// 2. read access caller has some roles like HIGHTPRIORY_REPLICA, NORMAL_REPLICA, CONSUMER, MAINTAIN_REPLIC
+// 3. each roles has different wight for read
+// 4. when the disk is busy that detected by system ioutil, it tell result for the caller by checkRead()
+// 5. each read can be allowed that the caller should get the token. the token controlled the throughput
+//    and concurrent by read for the caller.
+// 6. the background task to check the disk ioutil and return the token for user
+public class DiskLoadProtector implements Configurable {
+
+    // fetcher's roles
+    enum Role {
+        // the fetcher is replica that's fetch disk data when the broker bring up or the topic expand replica
+        maintainReplica,
+        // the fetcher indicate by replica id, if id equals -1 it should be consumer, otherwise is replcia
+        consumer,
+        // the fetcher is replica that's not maintain replica
+        normalReplica,
+        // the fetcher is replica that's selected from normal replica by disk load protector,
+        // and than give more bandwidth to handle it's read
+        highPrioryReplica,
+
+        MAX,
+    }
+
+    private static final Logger log = LoggerFactory.getLogger(DiskLoadProtector.class);
+
+    // weight of the roles.
+    private int maintainReplicaWeight = 1;
+    private int consumerWeight = 2;
+    private int normalReplicaWeight = 3;
+
+    // the rules generate by weight of roles
+    private Role[] rules = new Role[1024];
+
+    // ioutil check frequency (ms)
+    private int ioUtilFrequency = 10;
+    // max io capacity mb
+    private int maxIoCapacity = 100;
+
+    // allow max read request for one disk in one ioutil frequency
+    private int maxIoConcurrent = 3;
+
+    //enable or disable disk load protctor
+    private boolean enable = true;
+
+    // use mincore to check whether the read data is in page cache
+    private boolean accurateHotdataCheck = true;
+    // if the fetch request lag more than unreplicaLagMin and the read data is in disk. the replica called unreplica
+    private int unreplicaLagMin = 10000;
+
+    // use to select highPrioryTopicPartition from unrplicas
+    private ConcurrentHashMap<TopicPartitionAndReplica, UnreplicaTopicPartition> unReplicaTopicPartitions;
+    // the front of priory queue by these condiftion:
+    // 1. not maintain topicparitition and replica
+    // 2. lag is less than other
+    private PriorityQueue<UnreplicaTopicPartition> priorityQueue;
+    private TopicPartitionAndReplica highPrioryTopicPartition;
+    private ConcurrentHashSet<TopicPartitionAndReplica> maintainTopicPartitions;
+    private int cleanUnreplicaFrequncy = 5000;
+
+    //the token s used to limit throuput and concurrent for read on the disk
+    private HashMap<String, DiskToken> diskTokens;
+
+    private Random random;
+    private ScheduledExecutorService executorService;
+    private HashMap<String, DiskLoadProtectMetrics>  metrics;
+
+    public DiskLoadProtector() {
+    }
+
+    public DiskLoadProtector(ScheduledExecutorService executorService, boolean enable) {
+        this.enable = enable;
+        priorityQueue = new PriorityQueue<>();
+        unReplicaTopicPartitions = new ConcurrentHashMap<>();
+        maintainTopicPartitions = new ConcurrentHashSet<>();
+        random = new Random();
+
+        List<String> disks = OSUtil.instance().getAllDisks();
+        diskTokens = new HashMap<>();
+        for (String disk : disks) {
+            diskTokens.put(disk, new DiskToken(disk));
+        }
+
+        buildRules(maintainReplicaWeight, consumerWeight, normalReplicaWeight);
+
+        this.executorService = executorService;
+        createMetrics();
+        startSchedule();
+    }
+
+    private void startSchedule() {
+        // these flag need to set that check it support by os
+        accurateHotdataCheck = accurateHotdataCheck && OSUtil.instance().supportCheckFileInCache();
+        enable = enable && OSUtil.instance().supportIoUtil();
+
+        log.info("Disk load protector is {}, accurrate hot data check is {}, ioutil is {} supported",
+                enable ? "enable" : "disable", accurateHotdataCheck ? "enable" : "disable",
+                OSUtil.instance().supportIoUtil() ? "" : "not");
+
+        // start task for check ioutitl that monitor all disks. and the task return token by the disk's ioutil.
+        OSUtil.instance().setIoUtilFrequency(ioUtilFrequency);
+        executorService.schedule(() -> {
+            try {
+                checkDiskLoad();
+            } catch (Exception e) {
+                log.error("internal error: ", e);
+            }
+        }, ioUtilFrequency, TimeUnit.MILLISECONDS);
+
+        // start task for check unreplica active status. if not active remove it
+        executorService.scheduleWithFixedDelay(() -> {
+            try {
+                cleanUnreplica();
+            } catch (Exception e) {
+                log.error("internal error: ", e);
+            }
+        }, cleanUnreplicaFrequncy, cleanUnreplicaFrequncy, TimeUnit.MILLISECONDS);
+    }
+
+    void buildRules(int maintainWeight, int consumerWeight, int replicaWeight) {
+        int total = maintainWeight + consumerWeight + replicaWeight;
+        //init rules by weight of roles
+        for (int i = 0; i < rules.length; i++) {
+            int v = random.nextInt(total);
+            if (v < maintainWeight) {
+                rules[i] = Role.maintainReplica;
+            } else if (v < maintainWeight + consumerWeight) {
+                rules[i] = Role.consumer;
+            } else {
+                rules[i] = Role.normalReplica;
+            }
+        }
+    }
+
+    private void createMetrics() {
+        String group = "kafka.server";
+        String type = "DiskLoadProtector";
+
+        // using scala metric help class to keep the naming of metric is the same as kafka metrics.
+        var maintainTopics = new Gauge<String>() {
+            @Override
+            public String value() {
+                var elements = maintainTopicPartitions.stream().map(TopicPartitionAndReplica::toString);
+                return String.join(",", elements.collect(Collectors.toList()));
+            }
+        };
+        var tag = new LinkedHashMap<String, String>();
+        MetricName metricsName = KafkaExMetrics.createMetricsName(group, type, "MaintainTopics", tag);
+        KafkaExMetrics.createGauge(metricsName,  maintainTopics).mark();
+
+        var unreplicaTopics = new Gauge<String>() {
+            @Override
+            public String value() {
+                var elements = unReplicaTopicPartitions.keySet().stream().map(TopicPartitionAndReplica::toString);
+                return String.join(",", elements.collect(Collectors.toList()));
+            }
+        };
+        metricsName = KafkaExMetrics.createMetricsName(group, type, "UnreplicaTopics", tag);
+        KafkaExMetrics.createGauge(metricsName,  unreplicaTopics).mark();
+
+        var highPriorityTopic = new Gauge<String>() {
+            @Override
+            public String value() {
+                return highPrioryTopicPartition != null ?
+                    highPrioryTopicPartition.toString() : "";
+            }
+        };
+        metricsName = KafkaExMetrics.createMetricsName(group, type, "HighPriorityTopic", tag);
+        KafkaExMetrics.createGauge(metricsName,  highPriorityTopic).mark();
+
+        List<String> disks = OSUtil.instance().getAllDisks();
+        metrics = new HashMap<>();
+        for (String disk : disks) {
+            tag.put("disk", disk);
+            DiskLoadProtectMetrics diskLoadProtectMetrics = new DiskLoadProtectMetrics();
+            metricsName = KafkaExMetrics.createMetricsName(group, type, "DiskProctectTotal", tag);
+            diskLoadProtectMetrics.total = KafkaExMetrics.createMeter(metricsName, "disk protect total per second", TimeUnit.SECONDS);
+
+            metricsName = KafkaExMetrics.createMetricsName(group, type, "DiskProctectActive", tag);
+            diskLoadProtectMetrics.active = KafkaExMetrics.createMeter(metricsName, "disk protect perform per second", TimeUnit.SECONDS);
+            metrics.put(disk, diskLoadProtectMetrics);
+        }
+    }
+
+    // This function check all the disk current ioutil return token.
+    private void checkDiskLoad() {
+        List<String> disks = OSUtil.instance().getAllDisks();
+        int rate = (int)Math.round(maxIoCapacity / (1000.0 / ioUtilFrequency));
+        rate = rate == 0 ? 1 : rate;
+        // return token by diffrence ioutil.
+        // if ioutil is hight return less token, otherwise returen more tokens
+        for (String disk : disks) {
+            double value = OSUtil.instance().getIoUtil(disk);
+            if (value < 0.44) {
+                diskTokens.get(disk).returnToken(4 * rate);
+            } else if (value < 0.66) {
+                diskTokens.get(disk).returnToken(3 * rate);
+            } else if (value < 0.88) {
+                diskTokens.get(disk).returnToken(2 * rate);
+            } else if (value < 0.98)  {
+                diskTokens.get(disk).returnToken(rate);
+            }
+        }
+        log.trace("current disk token: {}, io status: {}", diskTokens, OSUtil.instance().getAllDiskStatus());
+
+        executorService.schedule(() -> {
+            try {
+                checkDiskLoad();
+            } catch (Exception e) {
+                log.error("internal error: ", e);
+            }
+        }, ioUtilFrequency, TimeUnit.MILLISECONDS);
+    }
+
+    // This function check all the unreplica topic partitions. it remove all the inactive replica from unreplicas
+    synchronized private void cleanUnreplica() {
+        var cleanUnreplicaTopicPartitions = new ArrayList<UnreplicaTopicPartition>();
+        for (UnreplicaTopicPartition unreplicaTopicPartition : unReplicaTopicPartitions.values()) {
+            if (System.currentTimeMillis() - unreplicaTopicPartition.timestamp > 60000) {
+                cleanUnreplicaTopicPartitions.add(unreplicaTopicPartition);
+            }
+        }
+
+        for (UnreplicaTopicPartition unreplicaTopicPartition : cleanUnreplicaTopicPartitions) {
+            log.info("disk protector remove unused unreplica topic: {}, lag: {}",
+                    unreplicaTopicPartition.topicPartitionAndReplica, unreplicaTopicPartition.lag);
+            removeUnreplicaTopicPartition(unreplicaTopicPartition.topicPartitionAndReplica);
+        }
+    }
+
+    public FetchInfo checkRead(FetchInfo fetchInfo) {
+        // empty read
+        if (fetchInfo.getPolicy() == FetchInfo.ReadPolicy.Empty) {
+            return fetchInfo;
+        }
+
+        //disk protector disable or os is not support
+        if (!enable) {
+            fetchInfo.setPolicy(FetchInfo.ReadPolicy.Normal);
+            return fetchInfo;
+        }
+
+        // check weather the read cause physical io, if the data in page cache return. otherwise continue
+        boolean hotdata = isHotdata(fetchInfo);
+        if (hotdata) {
+            fetchInfo.setPolicy(FetchInfo.ReadPolicy.Normal);
+            return fetchInfo;
+        }
+        // check read by policy
+        return doDiskProtect(fetchInfo);
+    }
+
+    private FetchInfo doDiskProtect(FetchInfo fetchInfo) {
+        assert !diskTokens.isEmpty();
+
+        String diskname = fetchInfo.getDiskname();
+        TopicPartitionAndReplica topicPartitionAndReplica = fetchInfo.getTopicPartitionAndReplica();
+        int replicaId = topicPartitionAndReplica.replicaId();
+        DiskToken token = diskTokens.get(diskname);
+
+        // each read allowd by get the token by role. when the token is empty it return Evade to give up the reading
+        // it the topic partition is hgih priory topic partiition return Greedy to read more data.
+        if (replicaId < 0) {
+            // the caller role is Consumer
+            boolean success = token.takeToken(Role.consumer);
+            DiskLoadProtectMetrics diskLoadProtectMetrics = metrics.get(diskname);
+            diskLoadProtectMetrics.total.meter().mark();
+            log.trace("disk protector perform action for consumer. topic: {}, disk: {}, token: {}, success: {}",
+                    topicPartitionAndReplica.topicPartition(), diskname, token, success);
+            if (!success) {
+                fetchInfo.setPolicy(FetchInfo.ReadPolicy.Evade);
+                diskLoadProtectMetrics.active.meter().mark();
+                return fetchInfo;
+            } else {
+                OSUtil.instance().loadPageCache(
+                        fetchInfo.getFilename(), fetchInfo.getLogOffset(), fetchInfo.getLength());
+                fetchInfo.setPolicy(FetchInfo.ReadPolicy.Normal);
+                return fetchInfo;
+            }
+        } else {
+            if (topicPartitionAndReplica.equals(highPrioryTopicPartition)) {
+                // the caller role is HightPriory replica
+                log.trace("{} topic: {}, replica: {}, disk: {}, token: {}, success: true",
+                        "disk protector perform action for high priory replica.",
+                        topicPartitionAndReplica.topicPartition(), replicaId, diskname, token);
+                token.takeToken(Role.highPrioryReplica);
+                fetchInfo.setPolicy(FetchInfo.ReadPolicy.Greedy);
+                fetchInfo.setLength(fetchInfo.getLength() * 4);
+                OSUtil.instance().loadPageCache(
+                        fetchInfo.getFilename(), fetchInfo.getLogOffset(), fetchInfo.getLength());
+                return fetchInfo;
+            }
+
+            // the caller role is noraml replica or maitain replica
+            boolean maintainTopicPartition = maintainTopicPartitions.contains(topicPartitionAndReplica);
+            boolean success = token.takeToken(maintainTopicPartition ? Role.maintainReplica : Role.normalReplica);
+            log.trace("disk protector perform action for {}replica. topic: {}, replica: {}, disk: {}, token: {}, success: {}",
+                    maintainTopicPartition ? "maintain " : "", topicPartitionAndReplica.topicPartition(),
+                    replicaId, diskname, token, success);
+            DiskLoadProtectMetrics diskLoadProtectMetrics = metrics.get(diskname);
+            diskLoadProtectMetrics.total.meter().mark();
+            if (!success) {
+                fetchInfo.setPolicy(FetchInfo.ReadPolicy.Evade);
+                diskLoadProtectMetrics.active.meter().mark();
+                return fetchInfo;
+            } else {
+                OSUtil.instance().loadPageCache(
+                        fetchInfo.getFilename(), fetchInfo.getLogOffset(), fetchInfo.getLength());
+                fetchInfo.setPolicy(FetchInfo.ReadPolicy.Normal);
+                return fetchInfo;
+            }
+        }
+    }
+
+    private boolean isHotdata(FetchInfo fetchInfo) {
+        String diskName = fetchInfo.getDiskname();
+        assert !diskName.isEmpty();
+
+        //normally the hotdata should be active log segment. if accurateHotdataCheck is enable check it by the mincore util
+        if (accurateHotdataCheck) {
+            boolean hotdata = OSUtil.instance().isCached(
+                    fetchInfo.getFilename(), fetchInfo.getLogOffset(), fetchInfo.getLength());
+            fetchInfo.setHotdata(hotdata);
+        }
+
+        if (fetchInfo.getTopicPartitionAndReplica().replicaId() >= 0) {
+            //caller is replica, need to check the replica status is unreplica or not
+            UnreplicaTopicPartition unreplicaTopicPartition = unReplicaTopicPartitions.get(fetchInfo.getTopicPartitionAndReplica());
+            if (unreplicaTopicPartition != null) {
+                // the replica in unreplicas.
+                // 1. update timestamp
+                // 2. if lag small enough, reomve it from unreplicas
+                unreplicaTopicPartition.timestamp = System.currentTimeMillis();
+                if (fetchInfo.getLag() < unreplicaLagMin) {
+                    log.info("disk protector remove unreplica topic: {}, lag: {}",
+                            fetchInfo.getTopicPartitionAndReplica(), fetchInfo.getLag());
+                    removeUnreplicaTopicPartition(fetchInfo.getTopicPartitionAndReplica());
+                }
+            } else {
+                // the replica not in unpeplicas. check wether it's read from disk and the lag is big enough to add to unreplicas
+                if (!fetchInfo.isHotdata() && fetchInfo.getLag() > unreplicaLagMin) {
+                    log.info("disk protector add unreplica topic: {}, lag: {}",
+                            fetchInfo.getTopicPartitionAndReplica(), fetchInfo.getLag());
+                    addUnreplicaTopicPartition(fetchInfo.getTopicPartitionAndReplica(), fetchInfo.getLag());
+                }
+            }
+        }
+        return fetchInfo.isHotdata();
+    }
+
+    synchronized public void addMaintainTopicPartitions(TopicPartitionAndReplica topicPartitionAndReplica) {
+        log.info("disk protector add maintain topic: {}", topicPartitionAndReplica);
+        if (maintainTopicPartitions.contains(topicPartitionAndReplica)) {
+            removeUnreplicaTopicPartition(topicPartitionAndReplica);
+        }
+        UnreplicaTopicPartition maintainTopicPartition = new UnreplicaTopicPartition(topicPartitionAndReplica, -1, true);
+        maintainTopicPartitions.add(topicPartitionAndReplica);
+        unReplicaTopicPartitions.put(topicPartitionAndReplica, maintainTopicPartition);
+    }
+
+    synchronized private void removeUnreplicaTopicPartition(TopicPartitionAndReplica topicPartitionAndReplica) {
+        maintainTopicPartitions.remove(topicPartitionAndReplica);
+        unReplicaTopicPartitions.remove(topicPartitionAndReplica);
+        priorityQueue.removeIf(
+            unreplicaTopicPartition -> unreplicaTopicPartition.topicPartitionAndReplica.equals(topicPartitionAndReplica)
+        );
+        if (topicPartitionAndReplica.equals(highPrioryTopicPartition)) {
+            setHightPrioryTopicPartition();
+        }
+    }
+
+    synchronized private void addUnreplicaTopicPartition(TopicPartitionAndReplica topicPartitionAndReplica, long lag) {
+        if (unReplicaTopicPartitions.containsKey(topicPartitionAndReplica)) {
+            removeUnreplicaTopicPartition(topicPartitionAndReplica);
+        }
+
+        UnreplicaTopicPartition unreplicaTopicPartition = new UnreplicaTopicPartition(topicPartitionAndReplica, lag, false);
+        unReplicaTopicPartitions.put(topicPartitionAndReplica, unreplicaTopicPartition);
+        priorityQueue.add(unreplicaTopicPartition);
+        if (highPrioryTopicPartition == null) {
+            setHightPrioryTopicPartition();
+        }
+    }
+
+    private void setHightPrioryTopicPartition() {
+        UnreplicaTopicPartition unreplicaTopicPartition = priorityQueue.peek();
+        if (unreplicaTopicPartition != null) {
+            highPrioryTopicPartition = unreplicaTopicPartition.topicPartitionAndReplica;
+            log.info("disk protector set hight priory topic: {}", highPrioryTopicPartition);
+        } else {
+            highPrioryTopicPartition = null;
+            log.info("disk protector set hight priory topic: None");
+        }
+    }
+
+    public int getIoUtilFrequency() {
+        return ioUtilFrequency;
+    }
+
+    @Override
+    public void configure(Map<String, ?> configs) {
+        if (configs.containsKey("enable")) {
+            enable = Boolean.parseBoolean((String) configs.get("enable"));
+        }
+        if (configs.containsKey("accurate.hotdata.enable")) {
+            accurateHotdataCheck = Boolean.parseBoolean((String) configs.get("accurate.hotdata.enable"));
+        }
+        if (configs.containsKey("unreplica.lag.min")) {
+            unreplicaLagMin = Integer.parseInt((String) configs.get("unreplica.lag.min"));
+        }
+        if (configs.containsKey("ioutil.frequency")) {
+            ioUtilFrequency = Integer.parseInt((String) configs.get("ioutil.frequency"));
+        }
+        OSUtil.instance().setIoUtilFrequency(ioUtilFrequency);
+
+        if (configs.containsKey("io.weight.normalreplica")) {
+            normalReplicaWeight = Integer.parseInt((String) configs.get("io.weight.normalreplica"));
+        }
+        if (configs.containsKey("io.weight.consumer")) {
+            consumerWeight = Integer.parseInt((String) configs.get("io.weight.consumer"));
+        }
+        if (configs.containsKey("io.weight.maintainreplcia")) {
+            maintainReplicaWeight = Integer.parseInt((String) configs.get("io.weight.maintainreplcia"));
+        }
+        if (configs.containsKey("io.max.concurrent")) {
+            maxIoConcurrent = Integer.parseInt((String) configs.get("io.max.concurrent"));
+        }
+        if (configs.containsKey("io.max.capacity")) {
+            maxIoCapacity = Integer.parseInt((String) configs.get("io.max.capacity"));
+        }
+
+        String configString = "Disk load protector config:\n";
+        configString += String.format("\tenable=%b\n", enable);
+        configString += String.format("\taccurate.hotdata.enable=%b\n", accurateHotdataCheck);
+        configString += String.format("\tunreplica.lag.min=%d\n", unreplicaLagMin);
+        configString += String.format("\tioutil.frequency=%d\n", ioUtilFrequency);
+        configString += String.format("\tio.weight.normalreplica=%d\n", normalReplicaWeight);
+        configString += String.format("\tio.weight.consumer=%d\n", consumerWeight);
+        configString += String.format("\tio.weight.maintainreplcia=%d\n", maintainReplicaWeight);
+        configString += String.format("\tio.max.concurrent=%d\n", maxIoConcurrent);
+        configString += String.format("\tio.max.capacity=%d\n", maxIoCapacity);
+        log.info(configString);
+
+        accurateHotdataCheck = accurateHotdataCheck && OSUtil.instance().supportCheckFileInCache();
+        enable = enable && OSUtil.instance().supportIoUtil();
+        buildRules(maintainReplicaWeight, consumerWeight, normalReplicaWeight);
+        log.info("Disk load protector is {}, accurrate hot data check is {}, ioutil is {} supported",
+                enable ? "enable" : "disable", accurateHotdataCheck ? "enable" : "disable",
+                OSUtil.instance().supportIoUtil() ? "" : "not");
+    }
+
+    public Properties getConfig() {
+        Properties properties = new Properties();
+        properties.setProperty("enable", String.valueOf(enable));
+        properties.setProperty("accurate.hotdata.enable", String.valueOf(enable));
+        properties.setProperty("unreplica.lag.min", String.valueOf(unreplicaLagMin));
+        properties.setProperty("ioutil.frequency", String.valueOf(ioUtilFrequency));
+        properties.setProperty("io.weight.normalreplica", String.valueOf(normalReplicaWeight));
+        properties.setProperty("io.weight.consumer", String.valueOf(consumerWeight));
+        properties.setProperty("io.weight.maintainreplcia", String.valueOf(maintainReplicaWeight));
+        properties.setProperty("io.max.concurrent", String.valueOf(maxIoConcurrent));
+        properties.setProperty("io.max.capacity", String.valueOf(maxIoCapacity));
+        return properties;
+    }
+
+    static class UnreplicaTopicPartition implements Comparable<UnreplicaTopicPartition> {
+        private TopicPartitionAndReplica topicPartitionAndReplica;
+        private boolean maintain;
+        private long lag;
+        private long timestamp;
+
+        UnreplicaTopicPartition(TopicPartitionAndReplica topicPartitionAndReplica, long lag, boolean maintain) {
+            this.topicPartitionAndReplica = topicPartitionAndReplica;
+            this.maintain = maintain;
+            this.lag = lag;
+            this.timestamp = System.currentTimeMillis();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj instanceof UnreplicaTopicPartition) {
+                UnreplicaTopicPartition utp = (UnreplicaTopicPartition) obj;
+                return topicPartitionAndReplica.equals(utp.topicPartitionAndReplica);
+            }
+            return false;
+        }
+
+        @Override
+        public int compareTo(UnreplicaTopicPartition o) {
+            if (maintain) {
+                if (maintain != o.maintain) {
+                    return 1;
+                }
+            } else {
+                if (o.maintain) {
+                    return -1;
+                }
+            }
+            return (int) (lag - o.lag);
+        }
+    }
+
+    static class DiskLoadProtectMetrics {
+        ExMeter total;
+        ExMeter active;
+    }
+
+    class DiskToken {
+        private String diskname;
+        //token array that indicate left token by role
+        private int[] tokens;
+
+        // the currentIndex indicate of rules than use for determine which role should be filled.
+        private int currentIndex;
+
+        //the timestap for the last time of get token by role
+        private long[] lastGetTokenTimestamp;
+
+        DiskToken(String diskname) {
+            this.diskname = diskname;
+            // init token for roles
+            tokens = new int[Role.MAX.ordinal()];
+            lastGetTokenTimestamp = new long[Role.MAX.ordinal()];
+            for (int i = 0; i < Role.MAX.ordinal(); i++) {
+                tokens[i] = 0;
+            }
+        }
+
+        boolean takeToken(Role role) {
+            boolean success = false;
+            synchronized (this) {
+                if (tokens[role.ordinal()] > 0 &&
+                        System.currentTimeMillis() - lastGetTokenTimestamp[role.ordinal()] > ioUtilFrequency / maxIoConcurrent) {
+                    tokens[role.ordinal()] -= 1;
+                    log.trace("disk protector take token for role: {}, disk: {}", role, diskname);
+                    lastGetTokenTimestamp[role.ordinal()] = System.currentTimeMillis();
+                    success = true;
+                }
+            }
+            return success;
+        }
+
+        void returnToken(int value) {
+            synchronized (this) {
+                // always fill high priory token if it is consumed
+                if (tokens[Role.highPrioryReplica.ordinal()] < 1) {
+                    tokens[Role.highPrioryReplica.ordinal()] = 1;
+                    log.trace("disk protector return token for role: highPrioryReplica, diskname: {}", diskname);
+                    value--;
+                }
+                int totalToken = 0;
+                for (int v : tokens) {
+                    totalToken += v;
+                }
+                // fill token by rules if it is consumed
+                while (value > 0 && totalToken < 3 * maxIoConcurrent + 1) {
+                    Role t = rules[currentIndex++];
+                    if (tokens[t.ordinal()] < maxIoConcurrent) {
+                        log.trace("disk protector return token for role: {}, diskname: {}", t, diskname);
+                        tokens[t.ordinal()]++;
+                        value--;
+                        totalToken++;
+                    }
+                    if (currentIndex == rules.length) {
+                        currentIndex = 0;
+                    }
+                }
+            }
+        }
+
+        public String toString() {
+            return String.format("%d, %d, %d, %d", tokens[0], tokens[1], tokens[2], tokens[3]);
+        }
+    }
+}
+
+

core/src/main/java/com/didichuxing/datachannel/kafka/server/FetchInfo.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.server;
+
+import kafka.server.LogOffsetMetadata;
+import org.apache.kafka.common.TopicPartition;
+
+public class FetchInfo {
+
+    public enum ReadPolicy {
+        // normally read
+        Normal,
+        // empty read
+        Empty,
+        // give up current reading and wait to next round
+        Evade,
+        // can read more data than normal
+        Greedy
+    }
+
+    private static final TopicPartitionAndReplica UnknownTopicOrPartitionAndReplica =
+            new TopicPartitionAndReplica(new TopicPartition("unknown", -1), -1);
+
+    //if fetcher is consumer the replica id is -1, otherwise is the broker id
+    private TopicPartitionAndReplica topicPartitionAndReplica = UnknownTopicOrPartitionAndReplica;
+    //fetch offset
+    private LogOffsetMetadata logOffsetMetadata;
+    // fetch length
+    private int length;
+    private String filename;
+    // the read policy indicate the caller read the data or not when the read data is be confirmed by log offset and length
+    private ReadPolicy policy;
+    private String diskname = OSUtil.DEFAULT_DISKNAME;
+    // the lag of fetch offset and log end offset
+    private long lag;
+
+    // hotdata is recently used data by read or write. it has been cached in page cache by os.
+    // normally the data in active log segment should be hotdata
+    // if the os support mincore, by this way it can get the status directly
+    private boolean hotdata;
+
+    public static FetchInfo EMPTY = new FetchInfo("unknown_file", LogOffsetMetadata.UnknownOffsetMetadata(), 0);
+
+    public FetchInfo(String filename, LogOffsetMetadata logOffsetMetadata, int length) {
+        this.filename = filename;
+        this.length = length;
+        this.logOffsetMetadata = logOffsetMetadata;
+        if (logOffsetMetadata == LogOffsetMetadata.UnknownOffsetMetadata()) {
+            this.policy = ReadPolicy.Empty;
+        } else {
+            this.policy = ReadPolicy.Normal;
+        }
+    }
+
+    public String getFilename() {
+        return filename;
+    }
+
+    public long getMessageOffset() {
+        return logOffsetMetadata.messageOffset();
+    }
+
+    public long getLogOffset() {
+        return logOffsetMetadata.relativePositionInSegment();
+    }
+
+    public void setLogOffsetMetadata(LogOffsetMetadata logOffsetMetadata) {
+        this.logOffsetMetadata = logOffsetMetadata;
+    }
+
+    public TopicPartitionAndReplica getTopicPartitionAndReplica() {
+        return topicPartitionAndReplica;
+    }
+
+    public void setTopicPartitionAndReplica(TopicPartitionAndReplica topicPartitionAndReplica) {
+        this.topicPartitionAndReplica = topicPartitionAndReplica;
+    }
+
+    public int getLength() {
+        return length;
+    }
+
+    public void setLength(int length) {
+        this.length = length;
+    }
+
+    public ReadPolicy getPolicy() {
+        return policy;
+    }
+
+    public void setPolicy(ReadPolicy policy) {
+        this.policy = policy;
+    }
+
+    public String getDiskname() {
+        return diskname;
+    }
+
+    public void setDiskname(String diskname) {
+        this.diskname = diskname;
+    }
+
+    public boolean isHotdata() {
+        return hotdata;
+    }
+
+    public void setHotdata(boolean hotdata) {
+        this.hotdata = hotdata;
+    }
+
+    public long getLag() {
+        return lag;
+    }
+
+    public void setLag(long lag) {
+        this.lag = lag;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("[file:%s, log offset:%s, length:%d, replica:%d, disk:%s, policy:%s, hotdata:%b",
+                filename, logOffsetMetadata, length, topicPartitionAndReplica.replicaId(), diskname, policy, hotdata);
+    }
+}
+
+

core/src/main/java/com/didichuxing/datachannel/kafka/server/LinuxMetrics.java

@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.server;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.RandomAccessFile;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+//This class read system status for /proc file system in linux operating system
+// disk status read by /proc/diskstats
+// networking status read by /proc/net/dev
+// memory status read by /proc/meminfo
+// cpu load status read by /proc/loadavg
+// cpu usage status read by /proc/stat
+// use "man proc" can lookup the more information for these files
+public class LinuxMetrics extends OSMetrics {
+
+    private static final Logger log = LoggerFactory.getLogger(LinuxMetrics.class);
+
+    private ScheduledExecutorService executorService;
+
+    // disk
+    private RandomAccessFile diskstatsFile;
+    // networking
+    private RandomAccessFile netFile;
+    // load avgerage
+    private RandomAccessFile loadavgFile;
+    // memory
+    private RandomAccessFile meminfoFile;
+    // cpu
+    private RandomAccessFile statFile;
+
+    // the disks io stat that read from /proc/diskstats by last time
+    private HashMap<String, DiskStats> lastAllDiskStats = new HashMap<>();
+
+    // the net io stat that read from /proc/net/dev by last time
+    private HashMap<String, NetStats> lastAllNetStats = new HashMap<>();
+
+    // the cpu stat that read from /proc/loadavg  and by last time
+    private CpuStats lastCpuStats = new CpuStats();
+
+
+    // the current disks status that compute by current disk stat and last disk stat in one cycle
+    private HashMap<String, DiskStatus> allDiskStatus = new HashMap<>();
+
+    // the current net status that compute by current net stat and last net stat in one cycle
+    private HashMap<String, NetStatus> allNetStatus =  new HashMap<>();
+
+    // the current memory status
+    private MemoryStatus memoryStatus = new MemoryStatus();
+
+    // the current cpu status that compute by current cpu stat and last cpu stat in one cycle
+    private CpuStatus cpuStatus = new CpuStatus();
+
+    public LinuxMetrics(ScheduledExecutorService executorService) throws Exception {
+        this.executorService = executorService;
+
+        //open /proc files
+        diskstatsFile = new RandomAccessFile("/proc/diskstats", "r");
+        netFile = new RandomAccessFile("/proc/net/dev", "r");
+        loadavgFile = new RandomAccessFile("/proc/loadavg", "r");
+        meminfoFile = new RandomAccessFile("/proc/meminfo", "r");
+        statFile = new RandomAccessFile("/proc/stat", "r");
+
+        updateSystemStatus();
+    }
+
+    private HashMap<String, DiskStats> readDiskStats() {
+        try {
+            HashMap<String, DiskStats> allDiskStats = new HashMap<>();
+            String line  = null;
+            diskstatsFile.seek(0);
+            while (null != (line = diskstatsFile.readLine())) {
+                String[] elments = line.trim().split("\\s+");
+                String diskName = elments[2];
+                if (!diskName.startsWith("sd") || diskName.length() != 3) {
+                    continue;
+                }
+                long read = Long.parseLong(elments[3]);
+                long readBytes = Long.parseLong(elments[5]) * 512;
+                long write = Long.parseLong(elments[7]);
+                long writeBytes = Long.parseLong(elments[9]) * 512;
+                long ioTime = Long.parseLong(elments[12]);
+                DiskStats diskStats = new DiskStats(read, readBytes, write, writeBytes, ioTime);
+                allDiskStats.put(diskName, diskStats);
+            }
+            return allDiskStats;
+        } catch (Exception e) {
+            log.error("read disk status exception: ", e);
+        }
+        return null;
+    }
+
+    private HashMap<String, NetStats> readNetStats() {
+        try {
+            HashMap<String, NetStats> allNetStats = new HashMap<>();
+            String line  = null;
+            netFile.seek(0);
+            int lineNum = 0;
+            while (null != (line = netFile.readLine())) {
+                lineNum++;
+                if (lineNum > 2) {
+                    String[] elments = line.trim().split("\\s+");
+                    String name = elments[0].replace(":", "");
+                    if (!name.startsWith("eth")) {
+                        continue;
+                    }
+                    long recvBytes = Long.parseLong(elments[1]);
+                    long sendBytes = Long.parseLong(elments[9]);
+                    NetStats netStats = new NetStats(sendBytes, recvBytes);
+                    allNetStats.put(name, netStats);
+                }
+            }
+            return allNetStats;
+        } catch (Exception e) {
+            log.error("read net status exception: ", e);
+        }
+        return null;
+    }
+
+
+    private void updateDiskStatus() {
+        HashMap<String, DiskStats> allDiskStats = readDiskStats();
+        assert allDiskStats != null;
+        for (Map.Entry<String, DiskStats> entry: allDiskStats.entrySet()) {
+            String diskname = entry.getKey();
+            DiskStats old = lastAllDiskStats.get(diskname);
+            if (old == null) {
+                old = new DiskStats();
+            }
+            DiskStatus ds = allDiskStatus.get(diskname);
+            if (ds != null) {
+                allDiskStatus.put(diskname, entry.getValue().computeDiskStatus(old));
+            } else {
+                allDiskStatus.put(diskname, new DiskStatus());
+            }
+        }
+        lastAllDiskStats = allDiskStats;
+    }
+
+    private void updateNetStatus() {
+        HashMap<String, NetStats> allNetStats = readNetStats();
+        assert allNetStats != null;
+        for (Map.Entry<String, NetStats> entry: allNetStats.entrySet()) {
+            String ethName = entry.getKey();
+            NetStats old = lastAllNetStats.get(ethName);
+            if (old == null) {
+                old = new NetStats();
+            }
+            NetStatus ds = allNetStatus.get(ethName);
+            if (ds != null) {
+                allNetStatus.put(ethName, entry.getValue().computeNetStatus(old));
+            } else {
+                allNetStatus.put(ethName, new NetStatus());
+            }
+        }
+        lastAllNetStats = allNetStats;
+    }
+
+
+    private void updateMemoryStatus() {
+        try {
+            String line  = null;
+            meminfoFile.seek(0);
+            line = meminfoFile.readLine(); //total
+            String[] elments = line.split("\\s+");
+            memoryStatus.total = Long.parseLong(elments[1]) / 1024.0;
+
+            line = meminfoFile.readLine(); //free
+            elments = line.split("\\s+");
+            memoryStatus.free = Long.parseLong(elments[1]) / 1024.0;
+
+            line = meminfoFile.readLine(); //buffer
+            elments = line.split("\\s+");
+            memoryStatus.buffer = Long.parseLong(elments[1]) / 1024.0;
+
+            line = meminfoFile.readLine(); //cached
+            elments = line.split("\\s+");
+            memoryStatus.cached = Long.parseLong(elments[1]) / 1024.0;
+            meminfoFile.readLine(); //swap cached
+
+            line = meminfoFile.readLine(); //active
+            elments = line.split("\\s+");
+            memoryStatus.used = Long.parseLong(elments[1]) / 1024.0;
+
+            line = meminfoFile.readLine(); //inactive
+            elments = line.split("\\s+");
+            memoryStatus.used += Long.parseLong(elments[1]) / 1024.0;
+        } catch (Exception e) {
+            log.error("read memroy info exception: ", e);
+        }
+    }
+
+    private void updateCpuStatus() {
+        try {
+            String line  = null;
+            loadavgFile.seek(0);
+            line = loadavgFile.readLine();
+            String[] elments = line.split("\\s+");
+            cpuStatus.load = Double.parseDouble(elments[0]);
+            cpuStatus.load5 = Double.parseDouble(elments[1]);
+            cpuStatus.load15 = Double.parseDouble(elments[2]);
+
+            CpuStats now  = readCpuStats();
+            assert now != null;
+            CpuStatus cs = now.computeCpuStatus(lastCpuStats);
+            cpuStatus.iowaitPercent = cs.iowaitPercent;
+            cpuStatus.userPercent = cs.userPercent;
+            cpuStatus.idlePercent = cs.idlePercent;
+            cpuStatus.sysPercent = cs.sysPercent;
+            lastCpuStats = now;
+        } catch (Exception e) {
+            log.error("read cpu info exception: ", e);
+        }
+    }
+
+    private CpuStats readCpuStats() {
+        try {
+            CpuStats cpuStats = new CpuStats();
+            statFile.seek(0);
+            String line = statFile.readLine();
+            String[] elments = line.split("\\s+");
+            cpuStats.user = Long.parseLong(elments[1]);
+            cpuStats.sys = Long.parseLong(elments[3]);
+            cpuStats.idle = Long.parseLong(elments[4]);
+            cpuStats.iowait = Long.parseLong(elments[5]);
+            elments[0] = "0";
+            cpuStats.total = Arrays.stream(elments).mapToLong(Long::parseLong).sum();
+            return cpuStats;
+        } catch (Exception e) {
+            log.error("read cpu info exception: ", e);
+        }
+        return null;
+    }
+
+    void updateSystemStatus() {
+        updateCpuStatus();
+        updateMemoryStatus();
+        updateDiskStatus();
+        updateNetStatus();
+
+        executorService.schedule(() -> {
+            try {
+                updateSystemStatus();
+            } catch (Exception e) {
+                log.error("read system status exception: ", e);
+            }
+        }, 1, TimeUnit.SECONDS);
+    }
+
+    public CpuStatus getCupStatus() {
+        return cpuStatus;
+    }
+
+    public MemoryStatus getMemoryStatus() {
+        return memoryStatus;
+    }
+
+    public NetStatus getNetStatus(String name) {
+        return allNetStatus.get(name);
+    }
+
+    public HashMap<String, NetStatus> getAllNetStatus() {
+        return allNetStatus;
+    }
+
+    public DiskStatus getDiskStatus(String name) {
+        return allDiskStatus.get(name);
+    }
+
+    public HashMap<String, DiskStatus> getAllDiskStatus() {
+        return allDiskStatus;
+    }
+
+    static class CpuStats {
+        public long user;
+        public long sys;
+        public long idle;
+        public long iowait;
+        public long total;
+
+        public CpuStatus computeCpuStatus(CpuStats old) {
+            CpuStatus cs = new CpuStatus();
+            double totalCpu = total - old.total;
+            if (totalCpu == 0) {
+                return cs;
+            }
+            cs.userPercent = (user - old.user) / totalCpu;
+            cs.idlePercent = (idle - old.idle) / totalCpu;
+            cs.sysPercent = (sys - old.sys) / totalCpu;
+            cs.iowaitPercent = (iowait - old.iowait) / totalCpu;
+            return cs;
+        }
+
+    }
+
+    static class NetStats {
+        private long sendBytes;
+        private long recvBytes;
+        private long timestamp;
+
+        public NetStats() {
+        }
+
+        public NetStats(long send, long recive) {
+            this.sendBytes = send;
+            this.recvBytes = recive;
+            this.timestamp = System.currentTimeMillis();
+        }
+
+        public NetStatus computeNetStatus(NetStats old) {
+            NetStatus ns = new NetStatus();
+            double time = (timestamp - old.timestamp) / 1000.0;
+            if (time == 0) return ns;
+
+            ns.sendPerSecond = (sendBytes - old.sendBytes) / 1048576.0 / time;
+            ns.recvPerSecond = (recvBytes - old.recvBytes) / 1048576.0 / time;
+            return ns;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("[%d,%d]", sendBytes, recvBytes);
+        }
+    }
+
+    static class DiskStats {
+        private long read;
+        private long readBytes;
+        private long write;
+        private long writeBytes;
+        private long ioTime;
+        private long timestamp;
+
+        public DiskStats() {
+        }
+
+        public DiskStats(long read, long readBytes, long write, long writeBytes, long ioTime) {
+            this.read = read;
+            this.readBytes = readBytes;
+            this.write = write;
+            this.writeBytes = writeBytes;
+            this.ioTime = ioTime;
+            this.timestamp = System.currentTimeMillis();
+        }
+
+        public DiskStatus computeDiskStatus(DiskStats old) {
+            DiskStatus ds = new DiskStatus();
+            double time = (timestamp - old.timestamp) / 1000.0;
+            if (time == 0) return ds;
+
+            ds.readPerSecond = (read - old.read) / time;
+            ds.readBytesPerSecond = (readBytes - old.readBytes) / 1048576.0 / time;
+            ds.writePerSecond = (write - old.write) / time;
+            ds.writeBytesPerSecond = (writeBytes - old.writeBytes) / 1048576.0 / time;
+            ds.ioUtil = ((double) ioTime - old.ioTime) / (timestamp - old.timestamp);
+            return ds;
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/server/OSMetrics.java

@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.server;
+
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.metrics.Measurable;
+import org.apache.kafka.common.metrics.Metrics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+
+public class OSMetrics {
+
+    private static final Logger log = LoggerFactory.getLogger(OSMetrics.class);
+
+    private DiskMetrics diskMetres;
+    private MemoryMetrics memoryMetrics;
+    private NetMetrics netMetrics;
+    private CpuMetrics cpuMetrics;
+
+    //The implementation for get system metrics
+    private OSMetrics osMetricsImpl;
+
+    public OSMetrics() {
+    }
+
+    public OSMetrics(Metrics metrics, ScheduledExecutorService executorService) throws Exception {
+        String metricsGroupName = "OSMetrics";
+
+        String os = System.getProperty("os.name");
+        if (os.toLowerCase().startsWith("linux")) {
+            osMetricsImpl = new LinuxMetrics(executorService);
+            diskMetres = new DiskMetrics(metrics, metricsGroupName);
+            memoryMetrics = new MemoryMetrics(metrics, metricsGroupName);
+            netMetrics = new NetMetrics(metrics, metricsGroupName);
+            cpuMetrics = new CpuMetrics(metrics, metricsGroupName);
+        } else {
+            log.warn("current os: {} not support system metrics", os);
+        }
+    }
+
+    public CpuStatus getCupStatus() {
+        return null;
+    }
+
+    public MemoryStatus getMemoryStatus() {
+        return null;
+    }
+
+    public NetStatus getNetStatus(String name) {
+        return null;
+    }
+
+    public HashMap<String, NetStatus> getAllNetStatus() {
+        return null;
+    }
+
+    public DiskStatus getDiskStatus(String name) {
+        return null;
+    }
+
+    public HashMap<String, DiskStatus> getAllDiskStatus() {
+        return null;
+    }
+
+    static class DiskStatus {
+        public double ioUtil;
+        public double readPerSecond;
+        public double writePerSecond;
+        public double readBytesPerSecond;
+        public double writeBytesPerSecond;
+
+        @Override
+        public String toString() {
+            return String.format("[%f,%f,%f,%f,%f]",
+                    readPerSecond, readBytesPerSecond, writePerSecond, writeBytesPerSecond, ioUtil);
+        }
+    }
+
+    static class NetStatus {
+        public double sendPerSecond;
+        public double recvPerSecond;
+
+        @Override
+        public String toString() {
+            return String.format("[%f,%f]", sendPerSecond, recvPerSecond);
+
+        }
+    }
+
+    static class CpuStatus {
+        public double load;
+        public double load5;
+        public double load15;
+        public double userPercent;
+        public double sysPercent;
+        public double idlePercent;
+        public double iowaitPercent;
+    }
+
+    static class MemoryStatus {
+        public double total;
+        public double free;
+        public double buffer;
+        public double cached;
+        public double used;
+    }
+
+    private class DiskMetrics {
+
+        public DiskMetrics(Metrics metrics, String metricGrpPrefix) {
+            HashMap<String, DiskStatus> allDiskStatus = osMetricsImpl.getAllDiskStatus();
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("dev", "disk");
+            for (Map.Entry<String, DiskStatus> entry: allDiskStatus.entrySet()) {
+                String diskname = entry.getKey();
+                metricsTags.put("name", diskname);
+
+                Measurable readBytesMeasure = (config, now) -> {
+                    DiskStatus ds = osMetricsImpl.getDiskStatus(diskname);
+                    return ds.readBytesPerSecond;
+                };
+                MetricName name = metrics.metricName("ReadBytesPerSecond", metricGrpPrefix,
+                        "The number of bytes to read per second", metricsTags);
+                metrics.addMetric(name, readBytesMeasure);
+
+                Measurable wrtieBytesMeasure = (config, now) -> {
+                    DiskStatus ds = osMetricsImpl.getDiskStatus(diskname);
+                    return ds.writeBytesPerSecond;
+                };
+                name = metrics.metricName("WriteBytesPerSecond", metricGrpPrefix,
+                        "The number of bytes to write per second", metricsTags);
+                metrics.addMetric(name, wrtieBytesMeasure);
+
+                Measurable readMeasure = (config, now) -> {
+                    DiskStatus ds = osMetricsImpl.getDiskStatus(diskname);
+                    return ds.readPerSecond;
+                };
+                name = metrics.metricName("ReadPerSecond", metricGrpPrefix,
+                        "The number of read per second", metricsTags);
+                metrics.addMetric(name, readMeasure);
+
+                Measurable writeMeasure = (config, now) -> {
+                    DiskStatus ds = osMetricsImpl.getDiskStatus(diskname);
+                    return ds.writePerSecond;
+                };
+                name = metrics.metricName("WritePerSecond", metricGrpPrefix,
+                        "The number of write per second", metricsTags);
+                metrics.addMetric(name, writeMeasure);
+
+                Measurable ioutilMeasure = (config, now) -> {
+                    DiskStatus ds = osMetricsImpl.getDiskStatus(diskname);
+                    return ds.ioUtil;
+                };
+                name = metrics.metricName("IoUtil", metricGrpPrefix,
+                        "The number of write per second", metricsTags);
+                metrics.addMetric(name, ioutilMeasure);
+            }
+        }
+    }
+
+    private class NetMetrics {
+
+        public NetMetrics(Metrics metrics, String metricGrpPrefix) {
+            HashMap<String, NetStatus> allNetStatus = osMetricsImpl.getAllNetStatus();
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("dev", "net");
+            for (Map.Entry<String, NetStatus> entry: allNetStatus.entrySet()) {
+                String ethname = entry.getKey();
+                metricsTags.put("name", ethname);
+
+                Measurable sendBytesMeasure = (config, now) -> {
+                    NetStatus ns = osMetricsImpl.getNetStatus(ethname);
+                    return ns.sendPerSecond;
+                };
+                MetricName name = metrics.metricName("SendBytesPerSecond", metricGrpPrefix,
+                        "The number of bytes to send per second", metricsTags);
+                metrics.addMetric(name, sendBytesMeasure);
+
+                Measurable recvBytesMeasure = (config, now) -> {
+                    NetStatus ns = osMetricsImpl.getNetStatus(ethname);
+                    return ns.recvPerSecond;
+                };
+                name = metrics.metricName("RecvBytesPerSecond", metricGrpPrefix,
+                        "The number of bytes to recv per second", metricsTags);
+                metrics.addMetric(name, recvBytesMeasure);
+            }
+        }
+    }
+
+    private class CpuMetrics {
+
+        public CpuMetrics(Metrics metrics, String metricGrpPrefix) {
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("dev", "cpu");
+
+            Measurable loadMeasure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.load;
+            };
+            MetricName name = metrics.metricName("loadavg", metricGrpPrefix,
+                    "load average for realtime", metricsTags);
+            metrics.addMetric(name, loadMeasure);
+
+            Measurable load5Measure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.load5;
+            };
+            name = metrics.metricName("load5avg", metricGrpPrefix,
+                    "load average for 5 minute", metricsTags);
+            metrics.addMetric(name, load5Measure);
+
+            Measurable load15Measure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.load15;
+            };
+            name = metrics.metricName("load15avg", metricGrpPrefix,
+                    "load average for 15 minute", metricsTags);
+            metrics.addMetric(name, load15Measure);
+
+            Measurable userMeasure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.userPercent;
+            };
+            name = metrics.metricName("user", metricGrpPrefix,
+                    "cpu user", metricsTags);
+            metrics.addMetric(name, userMeasure);
+
+            Measurable sysMeasure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.sysPercent;
+            };
+            name = metrics.metricName("sys", metricGrpPrefix,
+                    "cpu system", metricsTags);
+            metrics.addMetric(name, sysMeasure);
+
+            Measurable idleMeasure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.idlePercent;
+            };
+            name = metrics.metricName("idle", metricGrpPrefix,
+                    "cpu idle", metricsTags);
+            metrics.addMetric(name, idleMeasure);
+
+            Measurable iowaitMeasure = (config, now) -> {
+                CpuStatus cs = osMetricsImpl.getCupStatus();
+                return cs.iowaitPercent;
+            };
+            name = metrics.metricName("iowait", metricGrpPrefix,
+                    "cpu iowait", metricsTags);
+            metrics.addMetric(name, iowaitMeasure);
+        }
+    }
+
+    private class MemoryMetrics {
+
+        public MemoryMetrics(Metrics metrics, String metricGrpPrefix) {
+            Map<String, String> metricsTags = new LinkedHashMap<>();
+            metricsTags.put("dev", "memory");
+
+            Measurable totalMeasure = (config, now) -> {
+                MemoryStatus ms = osMetricsImpl.getMemoryStatus();
+                return ms.total;
+            };
+            MetricName name = metrics.metricName("total", metricGrpPrefix,
+                    "total memory for mb", metricsTags);
+            metrics.addMetric(name, totalMeasure);
+
+            Measurable bufferMeasure = (config, now) -> {
+                MemoryStatus ms = osMetricsImpl.getMemoryStatus();
+                return ms.buffer;
+            };
+            name = metrics.metricName("buffer", metricGrpPrefix,
+                    "buffer memory for mb", metricsTags);
+            metrics.addMetric(name, bufferMeasure);
+
+            Measurable cachedMeasure = (config, now) -> {
+                MemoryStatus ms = osMetricsImpl.getMemoryStatus();
+                return ms.cached;
+            };
+            name = metrics.metricName("cached", metricGrpPrefix,
+                    "cached memory for mb", metricsTags);
+            metrics.addMetric(name, cachedMeasure);
+
+            Measurable freeMeasure = (config, now) -> {
+                MemoryStatus ms = osMetricsImpl.getMemoryStatus();
+                return ms.free;
+            };
+            name = metrics.metricName("free", metricGrpPrefix,
+                    "free memory for mb", metricsTags);
+            metrics.addMetric(name, freeMeasure);
+
+            Measurable usedMeasure = (config, now) -> {
+                MemoryStatus ms = osMetricsImpl.getMemoryStatus();
+                return ms.used;
+            };
+            name = metrics.metricName("used", metricGrpPrefix,
+                    "used memory for mb", metricsTags);
+            metrics.addMetric(name, usedMeasure);
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/server/OSUtil.java

@@ -0,0 +1,648 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.server;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.EvictionListener;
+import com.googlecode.concurrentlinkedhashmap.Weighers;
+import com.sun.jna.Library;
+import com.sun.jna.Memory;
+import com.sun.jna.Native;
+import com.sun.jna.Pointer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sun.misc.Unsafe;
+import sun.nio.ch.FileChannelImpl;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class OSUtil {
+
+    private static final Logger log = LoggerFactory.getLogger(OSUtil.class);
+    public static final String DEFAULT_DISKNAME = "alldisks";
+
+    //test may be call start many times
+    private static boolean started = false;
+
+    enum OS {
+        Others,
+        Mac,
+        Linux
+    };
+
+    //os type
+    private OS osType;
+    // all used disk by configuration of log.dir
+    private List<DiskAndMountpoint> allDisks;
+    // the disks io stat that read from /proc/diskstats by last time
+    private HashMap<String, DiskStats> lastAllDiskStats;
+    // the current disks's status that compute by current disk stat and last disk stat in one cycle
+    private HashMap<String, DiskStatus> allDiskStatus;
+
+    private ScheduledExecutorService executorService;
+
+    private boolean supportIoUtil;
+    private int ioUtilFrequncy = 1000;
+
+    //Cache all the file's page info.
+    private PageInfoCache pageInfoCache;
+    //os page size
+    private int pageSize;
+
+    private static class OSUtilHolder {
+        private static final OSUtil INSTANCE = new OSUtil();
+    }
+
+    public static OSUtil instance() {
+        return OSUtilHolder.INSTANCE;
+    }
+
+    synchronized public static void start(List<String> dirs, ScheduledExecutorService executorService) throws Exception {
+        if (started) return;
+        OSUtilHolder.INSTANCE.init(dirs, executorService);
+        started = true;
+    }
+
+    private OSUtil() {
+        initOSType();
+        allDisks = new ArrayList<>();
+    }
+
+    private void init(List<String> dirs, ScheduledExecutorService executorService) throws Exception {
+        //init used disk by config : log.dirs
+        initUsedDisks(dirs);
+
+        //try to init iostat from os,if yes start schedule for monitor iostat
+        initDiskStats();
+        this.executorService = executorService;
+        if (osType == OS.Linux && supportIoUtil) {
+            pageSize = getNativePagesize();
+            pageInfoCache = new PageInfoCache(pageSize, 3000);
+
+            executorService.schedule(() -> {
+                try {
+                    computeDiskStatusForLinux();
+                } catch (Exception e) {
+                    log.error("compute disk status exception: ", e);
+                }
+            }, ioUtilFrequncy, TimeUnit.MILLISECONDS);
+        }
+    }
+
+    private void initOSType() {
+        String os = System.getProperty("os.name");
+        if (os.toLowerCase().startsWith("linux")) {
+            osType = OS.Linux;
+        } else if (os.toLowerCase().startsWith("mac")) {
+            osType = OS.Mac;
+        } else {
+            osType = OS.Others;
+        }
+        log.info("current os: {}", os);
+    }
+
+    private void initUsedDisks(List<String> dirs) throws Exception {
+        if (osType == OS.Linux) {
+            allDisks = readDiskInfoForLinux(dirs);
+        } else if (osType == OS.Mac) {
+            allDisks = readDiskInfoForMac(dirs);
+        }
+        log.info("all used disks :" + allDisks.stream().
+                map(DiskAndMountpoint::toString).collect(Collectors.joining(",")));
+    }
+
+    private void initDiskStats() {
+        allDiskStatus = new HashMap<>();
+        for (DiskAndMountpoint diskAndMountpoint : allDisks) {
+            allDiskStatus.put(diskAndMountpoint.diskName, new DiskStatus());
+        }
+        if (osType == OS.Linux) {
+            HashMap<String, DiskStats> currentDiskStats = readDiskStatsForLinux();
+            if (currentDiskStats != null) {
+                lastAllDiskStats = currentDiskStats;
+                //iostat should contain used disks
+                if (lastAllDiskStats.keySet().containsAll(allDiskStatus.keySet())) {
+                    supportIoUtil = true;
+                }
+            }
+        }
+        log.info("current system is {} support io status", supportIoUtil ? "" : "not");
+    }
+
+    double getIoUtil(String diskname) {
+        assert allDiskStatus != null;
+        DiskStatus diskStatus = allDiskStatus.get(diskname);
+        return diskStatus.ioUtil;
+    }
+
+    public HashMap<String, DiskStatus> getAllDiskStatus() {
+        assert allDiskStatus != null;
+        return allDiskStatus;
+    }
+
+    boolean supportCheckFileInCache() {
+        return supportIoUtil;
+    }
+
+    boolean supportIoUtil() {
+        return supportIoUtil;
+    }
+
+    //check the read file in page cache for give offset and length
+    boolean isCached(String filename, long offset, int length) {
+        return pageInfoCache.inPageCache(filename, offset, length);
+    }
+
+    void loadPageCache(String filename, long offset, int length) {
+        if (supportIoUtil) {
+            pageInfoCache.loadPageCache(filename, offset, length);
+        }
+    }
+
+
+    public String getDiskName(String filename) {
+        assert filename != null && !filename.isEmpty();
+        if (osType == OS.Linux || osType == OS.Mac) {
+            for (DiskAndMountpoint diskAndMountpoint : allDisks) {
+                if (filename.startsWith(diskAndMountpoint.mountpoint)) {
+                    return diskAndMountpoint.diskName;
+                }
+            }
+        }
+        return DEFAULT_DISKNAME;
+    }
+
+    List<String> getAllDisks() {
+        List<String> allDiskNames = new ArrayList<>();
+        allDisks.forEach(diskAndMountpoint -> {
+            allDiskNames.add(diskAndMountpoint.diskName);
+        });
+        return allDiskNames;
+    }
+
+    private List<DiskAndMountpoint> readDiskInfoForLinux(List<String> dirs) throws Exception {
+        List<DiskAndMountpoint> diskMountPoints = new ArrayList<>();
+        String result = execSystemCommand("df");
+        String reg = "^/dev/([\\w]+)\\d\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+([^\\s]+)$";
+        Pattern pattern = Pattern.compile(reg, Pattern.MULTILINE);
+        assert result != null;
+        Matcher matcher = pattern.matcher(result);
+        DiskAndMountpoint root = new DiskAndMountpoint(DEFAULT_DISKNAME, "/");
+        while (matcher.find()) {
+            String mountDir = matcher.group(2);
+            String diskName = matcher.group(1);
+            if ("/".equals(mountDir)) {
+                root = new DiskAndMountpoint(diskName, mountDir);
+                continue;
+            }
+            dirs.forEach(dir -> {
+                if (dir.startsWith(mountDir)) {
+                    diskMountPoints.add(new DiskAndMountpoint(matcher.group(1), matcher.group(2)));
+                }
+            });
+        }
+        if (diskMountPoints.isEmpty()) {
+            diskMountPoints.add(root);
+        }
+        diskMountPoints.sort((x, y) -> y.mountpoint.length() - x.mountpoint.length());
+        assert !diskMountPoints.isEmpty();
+        return diskMountPoints;
+    }
+
+
+    private List<DiskAndMountpoint> readDiskInfoForMac(List<String> dirs) throws Exception {
+        List<DiskAndMountpoint> diskMountPoints = new ArrayList<>();
+        String result = execSystemCommand("df");
+        String reg = "^/dev/([\\w]+)\\d\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+[^\\s]+\\s+([^\\s]+)$";
+        Pattern pattern = Pattern.compile(reg, Pattern.MULTILINE);
+        assert result != null;
+        Matcher matcher = pattern.matcher(result);
+        DiskAndMountpoint root = new DiskAndMountpoint(DEFAULT_DISKNAME, "/");
+        while (matcher.find()) {
+            String mountDir = matcher.group(2);
+            String diskName = matcher.group(1);
+            if ("/".equals(mountDir)) {
+                root = new DiskAndMountpoint(diskName, mountDir);
+                continue;
+            }
+            dirs.forEach(dir -> {
+                if (dir.startsWith(mountDir)) {
+                    diskMountPoints.add(new DiskAndMountpoint(matcher.group(1), matcher.group(2)));
+                }
+            });
+        }
+        if (diskMountPoints.isEmpty()) {
+            diskMountPoints.add(root);
+        }
+        diskMountPoints.sort((x, y) -> y.mountpoint.length() - x.mountpoint.length());
+        assert !diskMountPoints.isEmpty();
+        return diskMountPoints;
+    }
+
+    private HashMap<String, DiskStats> readDiskStatsForLinux() {
+        try {
+            HashMap<String, DiskStats> allDiskStats = new HashMap<>();
+            String result = readFile("/proc/diskstats");
+            String[] lines = result.split("\n");
+            for (String line : lines) {
+                String[] elments = line.substring(13).split(" ");
+                String diskName = elments[0];
+                if (allDiskStatus.containsKey(diskName)) {
+                    long read = Long.parseLong(elments[1]);
+                    long readBytes = Long.parseLong(elments[3]) * 512;
+                    long write = Long.parseLong(elments[5]);
+                    long writeBytes = Long.parseLong(elments[7]) * 512;
+                    long ioTime = Long.parseLong(elments[10]);
+                    DiskStats diskStats = new DiskStats(read, readBytes, write, writeBytes, ioTime);
+                    allDiskStats.put(diskName, diskStats);
+                }
+            }
+            return allDiskStats;
+        } catch (Exception e) {
+            log.error("read disk status exception: ", e);
+        }
+        return null;
+    }
+
+    private void computeDiskStatusForLinux() {
+        HashMap<String, DiskStats> currentDiskStats = readDiskStatsForLinux();
+        assert currentDiskStats != null;
+        assert currentDiskStats.size() == allDiskStatus.size();
+
+        for (Map.Entry<String, DiskStatus> entry: allDiskStatus.entrySet()) {
+            String diskname = entry.getKey();
+            DiskStats now = currentDiskStats.get(diskname);
+            if (now == null) {
+                now = new DiskStats();
+            }
+            DiskStats old = lastAllDiskStats.get(diskname);
+            now.computeDiskStatus(old, entry.getValue());
+        }
+        lastAllDiskStats = currentDiskStats;
+
+        executorService.schedule(() -> {
+            try {
+                computeDiskStatusForLinux();
+            } catch (Exception e) {
+                log.error("compute disk status exception: ", e);
+            }
+        }, ioUtilFrequncy, TimeUnit.MILLISECONDS);
+    }
+
+    public void setIoUtilFrequency(int ioUtilFrequncy) {
+        this.ioUtilFrequncy = ioUtilFrequncy;
+    }
+
+    //for testing
+    public void dropPageCache()  {
+        if (osType == OS.Linux)
+            execSystemCommand("echo 1 > /proc/sys/vm/drop_caches");
+    }
+
+    private String execSystemCommand(String command) {
+        try {
+            String[] cmd = {"/bin/sh", "-c", command};
+            Process process = Runtime.getRuntime().exec(cmd);
+            InputStream inputStream = process.getInputStream();
+            byte[] bytes = inputStream.readAllBytes();
+            String resutl = new String(bytes, Charset.defaultCharset());
+            inputStream.close();
+
+            inputStream = process.getErrorStream();
+            bytes = inputStream.readAllBytes();
+            String error = new String(bytes, Charset.defaultCharset());
+            log.info("command:{}\nstdout:\n{}stderr:\n{}", command, resutl, error);
+            inputStream.close();
+            return resutl;
+        } catch (Exception e) {
+            log.error("execute system command:{}\nexception:\n{}", command, e);
+        }
+        return null;
+    }
+
+
+    private String readFile(String filename) {
+        try {
+            FileInputStream inputStream = new FileInputStream(new File(filename));
+            byte[] bytes = inputStream.readAllBytes();
+            String resutl = new String(bytes, Charset.defaultCharset());
+            inputStream.close();
+            return resutl;
+        } catch (Exception e) {
+            log.error("read file {} exception: ", filename, e);
+        }
+        return "";
+    }
+
+    static class DiskAndMountpoint {
+        String diskName;
+        String mountpoint;
+
+        DiskAndMountpoint(String diskName, String mountpoint) {
+            this.diskName = diskName;
+            this.mountpoint = mountpoint;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("[%s,%s]", diskName, mountpoint);
+        }
+    }
+
+    static class DiskStatus {
+        private double ioUtil;
+        private double readPerSecond;
+        private double writePerSecond;
+        private double readBytesPerSecond;
+        private double writeBytesPerSecond;
+
+        @Override
+        public String toString() {
+            return String.format("[%f,%f,%f,%f,%f]",
+                    readPerSecond, readBytesPerSecond, writePerSecond, writeBytesPerSecond, ioUtil);
+        }
+    }
+
+    static class DiskStats {
+        private long read;
+        private long readBytes;
+        private long write;
+        private long writeBytes;
+        private long ioTime;
+        private long timestamp;
+
+        public DiskStats() {
+        }
+
+        public DiskStats(long read, long readBytes, long write, long writeBytes, long ioTime) {
+            this.read = read;
+            this.readBytes = readBytes;
+            this.write = write;
+            this.writeBytes = writeBytes;
+            this.ioTime = ioTime;
+            this.timestamp = System.currentTimeMillis();
+        }
+
+        public void computeDiskStatus(DiskStats old, DiskStatus ds) {
+            double time = (timestamp - old.timestamp) / 1000.0;
+            if (time == 0) return;
+
+            ds.readPerSecond = (read - old.read) / time;
+            ds.readBytesPerSecond = (readBytes - old.readBytes) / time;
+            ds.writePerSecond = (write - old.write) / time;
+            ds.writeBytesPerSecond = (writeBytes - old.writeBytes) / time;
+            ds.ioUtil = ((double) ioTime - old.ioTime) / (timestamp - old.timestamp);
+        }
+
+        @Override
+        public String toString() {
+            return String.format("[%d,%d,%d,%d,%d]",
+                    read, readBytes, write, writeBytes, ioTime);
+        }
+    }
+
+    // This is the idiomatic JNA way of dealing with native code
+    // it is also possible to use native JNA methods
+    // and alternatively JNI
+    public interface CLibray extends Library {
+        CLibray INSTANCE = (CLibray) Native.load("c", CLibray.class);
+
+        // int mincore(void *addr, size_t length, unsigned char *vec);
+        // JNA will automagically convert the parameters to native parameters
+        int mincore(Pointer addr, long length, Pointer vec);
+    }
+
+    private static int getNativePagesize() throws Exception {
+        // we can do this with reflection
+        Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe");
+        unsafeField.setAccessible(true);
+        Unsafe unsafe = (Unsafe) unsafeField.get(null);
+        return unsafe.pageSize();
+    }
+
+    public static void unmap(final MappedByteBuffer buffer) throws Exception {
+        if (null != buffer) {
+            final Method method = FileChannelImpl.class.getDeclaredMethod("unmap", MappedByteBuffer.class);
+            method.setAccessible(true);
+            method.invoke(null, buffer);
+        }
+    }
+
+    // This class used to lookup the read data of the file is in the pagecahe or not
+    // the data in cache means the most of pages for read data is in the page cache.
+    private class PageInfo {
+
+        private static final int RefreshFileSize = 100 * 1024 * 1024;
+        private static final int RefreshTimeMs = 30 * 1000;
+        private final String filename;
+        private final FileChannel fc;
+        private MappedByteBuffer mmaped;
+        private AtomicBoolean inRefresh = new AtomicBoolean(false);
+        private ReentrantReadWriteLock rwlock = new ReentrantReadWriteLock();
+
+        // the array indicate each page of the file is in pagecache or not.
+        private byte[] pageInfo = new byte[0];
+        // page info load timestamp
+        private long timestamp = 0;
+        private long filesize = -RefreshFileSize;
+
+        public PageInfo(String filename) throws Exception {
+            // calls mmap
+            this.filename = filename;
+            fc = FileChannel.open(Paths.get(filename), StandardOpenOption.READ);
+            refreshPageInfo();
+        }
+
+        public void close() throws Exception {
+           fc.close();
+           unmap(mmaped);
+        }
+
+        synchronized void refreshPageInfo() throws Exception {
+            if (System.currentTimeMillis() - timestamp < RefreshTimeMs) {
+                return;
+            }
+
+            ReentrantReadWriteLock.WriteLock lock = rwlock.writeLock();
+            lock.lock();
+            try {
+                long size = fc.size();
+                if (size - filesize >= RefreshFileSize) {
+                    unmap(mmaped);
+                    mmaped = fc.map(FileChannel.MapMode.READ_ONLY, 0, size);
+                    filesize = size;
+                }
+            } finally {
+                lock.unlock();
+            }
+
+            // we need to prepare the vec array that will hold the results
+            // the vec must point to an array containing at least (length+PAGE_SIZE-1) / PAGE_SIZE bytes
+            int pages = (int) ((filesize + (pageSize - 1)) / pageSize);
+            Memory vec = new Memory(pages);
+            // do the mincore call using JNA (we could just as well used JNI)
+            Pointer mapPointer = Native.getDirectBufferPointer(mmaped);
+            int result = CLibray.INSTANCE.mincore(mapPointer, filesize, vec);
+            if (result != 0) {
+                throw new RuntimeException("Call to mincore failed with return value " + result);
+            }
+            if (pageInfo.length == pages) {
+                vec.read(0, pageInfo, 0, pages);
+            } else {
+                pageInfo = vec.getByteArray(0, pages);
+            }
+            log.debug("load file {} page info, {}", filename, this);
+            timestamp = System.currentTimeMillis();
+            inRefresh.set(false);
+        }
+
+        boolean inPageCache(long offset, int length, int pagesize) {
+            if (System.currentTimeMillis() - timestamp > RefreshTimeMs &&
+                    !inRefresh.compareAndExchange(false, true)) {
+                executorService.execute(() -> {
+                    try {
+                        refreshPageInfo();
+                    } catch (Exception e) {
+                        log.error("refresh pageinfo exception: ", e);
+                    }
+                });
+            }
+            byte[] currentPageInfo = pageInfo;
+            int start = (int) (offset / pagesize);
+            int end = (int) (offset + length) / pagesize;
+            int tail = Math.min(end, currentPageInfo.length);
+            int total = end - start;
+            int outMemoryCount = 0;
+            for (int i = start; i < tail; i++) {
+                if (currentPageInfo[i] != 1) {
+                    outMemoryCount++;
+                }
+            }
+            // the 80% pages in page cache should regconize in cache
+            return (double) outMemoryCount / total <= 0.2;
+        }
+
+        void loadPageCache(long offset, int length, int pagesize) throws Exception {
+            ReentrantReadWriteLock.ReadLock lock = rwlock.readLock();
+            try {
+                lock.lock();
+                long end = Math.min(offset + length, filesize);
+                for (int i = (int) offset; i < end; i += pagesize) {
+                    mmaped.get(i);
+                }
+            } finally {
+                lock.unlock();
+            }
+        }
+
+        @Override
+        public String toString() {
+            int count = 0;
+            for (byte b : pageInfo) {
+                if (b == 1) {
+                    count++;
+                }
+            }
+            return String.format("file size: %d, total pages: %d, in cache pages: %d, rate: %f",
+                    filesize, pageInfo.length, count, count * 1.0 / pageInfo.length);
+        }
+    }
+
+    // This class cached maximum pageinfos by filename. the fileinfo record the status of file pages in pagecache
+    // the life cycle of pageinfo is 30 seconds. when someone detect it has expired. it should reload the pageinfo.
+    // the cache use lru cache, when the lru cache is full. it drop the Least recently used entry from cache and load new one.
+    private class PageInfoCache {
+
+        private ConcurrentLinkedHashMap<String, PageInfo> pageInfos;
+        private int pagesize;
+
+        PageInfoCache(int pagesize, int maxSize) {
+            pageInfos = new ConcurrentLinkedHashMap.Builder<String, PageInfo>().
+                    maximumWeightedCapacity(maxSize).weigher(Weighers.singleton()).
+                    listener(new EvictionListener<String, PageInfo>() {
+                        @Override
+                        public void onEviction(String key, PageInfo value) {
+                            try {
+                                value.close();
+                            } catch (Exception e) {
+                                log.error("close pageinfo exception: ", e);
+                            }
+                        }
+                    }).build();
+            this.pagesize = pagesize;
+        }
+
+        boolean inPageCache(String filename, long offset, int length) {
+            try {
+                PageInfo pageInfo = pageInfos.get(filename);
+                if (pageInfo == null) {
+                    pageInfo = getPageInfo(filename);
+                }
+                return pageInfo.inPageCache(offset, length, pagesize);
+            } catch (Exception e) {
+                log.error("check inCache exception: ", e);
+                return false;
+            }
+        }
+
+        void loadPageCache(String filename, long offset, int length) {
+            try {
+                PageInfo pageInfo = pageInfos.get(filename);
+                if (pageInfo == null) {
+                    pageInfo = getPageInfo(filename);
+                }
+                pageInfo.loadPageCache(offset, length, pagesize);
+            } catch (Exception e) {
+                log.error("load pagecache exception: ", e);
+            }
+        }
+
+        synchronized private PageInfo getPageInfo(String filename) throws Exception {
+            //try to get pageinfo from cache again. it maybe get it from other thread loaded.
+            PageInfo pageInfo = pageInfos.get(filename);
+            if (pageInfo != null) {
+                return pageInfo;
+            } else {
+                pageInfo = new PageInfo(filename);
+                pageInfos.put(filename, pageInfo);
+            }
+            return pageInfo;
+        }
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/server/TopicPartitionAndReplica.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.server;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.Topic;
+
+public class TopicPartitionAndReplica {
+
+    final private TopicPartition topicPartition;
+    final private int replicaId;
+
+    public TopicPartitionAndReplica(TopicPartition topicPartition, int replicaId) {
+        this.topicPartition = topicPartition;
+        this.replicaId = replicaId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s:%d", topicPartition, replicaId);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        TopicPartitionAndReplica that =  o instanceof TopicPartitionAndReplica ? (TopicPartitionAndReplica) o : null;
+        if (that == null) {
+            return false;
+        } else {
+            return topicPartition.equals(that.topicPartition) && replicaId == that.replicaId;
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return topicPartition.hashCode() + 31 * replicaId;
+    }
+
+    public int replicaId() {
+        return replicaId;
+    }
+
+    public TopicPartition topicPartition() {
+        return topicPartition;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/util/KafkaUtils.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaUtils {
+    static private HashMap<Integer, String> fetchApiVersionMap = new HashMap<>();
+    static private HashMap<Integer, String> produceApiVersionMap = new HashMap<>();
+
+    static {
+        fetchApiVersionMap.put(0, "0.8");
+        fetchApiVersionMap.put(1, "0.9");
+        fetchApiVersionMap.put(2, "0.10.x");
+        fetchApiVersionMap.put(3, "0.10.x");
+        fetchApiVersionMap.put(4, "0.11");
+        fetchApiVersionMap.put(5, "0.11");
+        fetchApiVersionMap.put(6, "1.x");
+        fetchApiVersionMap.put(7, "1.x");
+        fetchApiVersionMap.put(8, "2.0");
+        fetchApiVersionMap.put(9, "2.0");
+        fetchApiVersionMap.put(10, "2.2");
+        fetchApiVersionMap.put(11, "2.4");
+
+        produceApiVersionMap.put(0, "0.8");
+        produceApiVersionMap.put(1, "0.9");
+        produceApiVersionMap.put(2, "0.10.x");
+        produceApiVersionMap.put(3, "0.11");
+        produceApiVersionMap.put(4, "1.x");
+        produceApiVersionMap.put(5, "1.x");
+        produceApiVersionMap.put(6, "2.0");
+        produceApiVersionMap.put(7, "2.2");
+        produceApiVersionMap.put(8, "2.4");
+    }
+
+    static public String apiVersionToKafkaVersion(int apiKey, int apiVersion) {
+        String result = null;
+        switch (apiKey) {
+            case 0:
+                result = produceApiVersionMap.get(apiVersion);
+                break;
+            case 1:
+                result = fetchApiVersionMap.get(apiVersion);
+        }
+        if (result == null) {
+            result = "unknown";
+        }
+        return result;
+    }
+}

core/src/main/java/com/didichuxing/datachannel/kafka/util/ScalaUtil.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.didichuxing.datachannel.kafka.util;
+
+import scala.Option;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import java.util.List;
+
+public class ScalaUtil {
+
+    public static <T> Seq<T> toSeq(List<T> list) {
+        return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq();
+    }
+
+    public static <T> Option<T> toOption(T t) {
+        return Option.apply(t);
+    }
+}

core/src/main/scala/kafka/Kafka.scala

@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka
+
+import java.util.Properties
+
+import joptsimple.OptionParser
+import kafka.utils.Implicits._
+import kafka.server.{KafkaServer, KafkaServerStartable}
+import kafka.utils.{CommandLineUtils, Exit, Logging}
+import org.apache.kafka.common.utils.{Java, LoggingSignalHandler, OperatingSystem, Utils}
+
+import scala.collection.JavaConverters._
+
+object Kafka extends Logging {
+
+  def getPropsFromArgs(args: Array[String]): Properties = {
+    val optionParser = new OptionParser(false)
+    val overrideOpt = optionParser.accepts("override", "Optional property that should override values set in server.properties file")
+      .withRequiredArg()
+      .ofType(classOf[String])
+    // This is just to make the parameter show up in the help output, we are not actually using this due the
+    // fact that this class ignores the first parameter which is interpreted as positional and mandatory
+    // but would not be mandatory if --version is specified
+    // This is a bit of an ugly crutch till we get a chance to rework the entire command line parsing
+    optionParser.accepts("version", "Print version information and exit.")
+
+    if (args.length == 0 || args.contains("--help")) {
+      CommandLineUtils.printUsageAndDie(optionParser, "USAGE: java [options] %s server.properties [--override property=value]*".format(classOf[KafkaServer].getSimpleName()))
+    }
+
+    if (args.contains("--version")) {
+      CommandLineUtils.printVersionAndDie()
+    }
+
+    val props = Utils.loadProps(args(0))
+
+    if (args.length > 1) {
+      val options = optionParser.parse(args.slice(1, args.length): _*)
+
+      if (options.nonOptionArguments().size() > 0) {
+        CommandLineUtils.printUsageAndDie(optionParser, "Found non argument parameters: " + options.nonOptionArguments().toArray.mkString(","))
+      }
+
+      props ++= CommandLineUtils.parseKeyValueArgs(options.valuesOf(overrideOpt).asScala)
+    }
+    props
+  }
+
+  def main(args: Array[String]): Unit = {
+    try {
+      val serverProps = getPropsFromArgs(args)
+      val kafkaServerStartable = KafkaServerStartable.fromProps(serverProps)
+
+      try {
+        if (!OperatingSystem.IS_WINDOWS && !Java.isIbmJdk)
+          new LoggingSignalHandler().register()
+      } catch {
+        case e: ReflectiveOperationException =>
+          warn("Failed to register optional signal handler that logs a message when the process is terminated " +
+            s"by a signal. Reason for registration failure is: $e", e)
+      }
+
+      // attach shutdown handler to catch terminating signals as well as normal termination
+      Exit.addShutdownHook("kafka-shutdown-hook", kafkaServerStartable.shutdown)
+
+      kafkaServerStartable.startup()
+      kafkaServerStartable.awaitShutdown()
+    }
+    catch {
+      case e: Throwable =>
+        fatal("Exiting Kafka due to fatal exception", e)
+        Exit.exit(1)
+    }
+    Exit.exit(0)
+  }
+}

core/src/main/scala/kafka/admin/AclCommand.scala

@@ -0,0 +1,660 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.util.Properties
+
+import joptsimple._
+import joptsimple.util.EnumConverter
+import kafka.security.authorizer.{AclAuthorizer, AclEntry, AuthorizerUtils}
+import kafka.server.KafkaConfig
+import kafka.utils._
+import org.apache.kafka.clients.admin.{Admin, AdminClientConfig}
+import org.apache.kafka.common.acl._
+import org.apache.kafka.common.acl.AclOperation._
+import org.apache.kafka.common.acl.AclPermissionType.{ALLOW, DENY}
+import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourcePatternFilter, Resource => JResource, ResourceType => JResourceType}
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.security.auth.KafkaPrincipal
+import org.apache.kafka.common.utils.{Utils, SecurityUtils => JSecurityUtils}
+import org.apache.kafka.server.authorizer.Authorizer
+
+import scala.collection.JavaConverters._
+import scala.compat.java8.OptionConverters._
+import scala.collection.mutable
+import scala.io.StdIn
+
+object AclCommand extends Logging {
+
+  val ClusterResourceFilter = new ResourcePatternFilter(JResourceType.CLUSTER, JResource.CLUSTER_NAME, PatternType.LITERAL)
+
+  private val Newline = scala.util.Properties.lineSeparator
+
+  def main(args: Array[String]): Unit = {
+
+    val opts = new AclCommandOptions(args)
+
+    CommandLineUtils.printHelpAndExitIfNeeded(opts, "This tool helps to manage acls on kafka.")
+
+    opts.checkArgs()
+
+    val aclCommandService = {
+      if (opts.options.has(opts.bootstrapServerOpt)) {
+        new AdminClientService(opts)
+      } else {
+        val authorizerClassName = if (opts.options.has(opts.authorizerOpt))
+          opts.options.valueOf(opts.authorizerOpt)
+        else
+          classOf[AclAuthorizer].getName
+
+        new AuthorizerService(authorizerClassName, opts)
+      }
+    }
+
+    try {
+      if (opts.options.has(opts.addOpt))
+        aclCommandService.addAcls()
+      else if (opts.options.has(opts.removeOpt))
+        aclCommandService.removeAcls()
+      else if (opts.options.has(opts.listOpt))
+        aclCommandService.listAcls()
+    } catch {
+      case e: Throwable =>
+        println(s"Error while executing ACL command: ${e.getMessage}")
+        println(Utils.stackTrace(e))
+        Exit.exit(1)
+    }
+  }
+
+  sealed trait AclCommandService {
+    def addAcls(): Unit
+    def removeAcls(): Unit
+    def listAcls(): Unit
+  }
+
+  class AdminClientService(val opts: AclCommandOptions) extends AclCommandService with Logging {
+
+    private def withAdminClient(opts: AclCommandOptions)(f: Admin => Unit): Unit = {
+      val props = if (opts.options.has(opts.commandConfigOpt))
+        Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+      else
+        new Properties()
+      props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+      val adminClient = Admin.create(props)
+
+      try {
+        f(adminClient)
+      } finally {
+        adminClient.close()
+      }
+    }
+
+    def addAcls(): Unit = {
+      val resourceToAcl = getResourceToAcls(opts)
+      withAdminClient(opts) { adminClient =>
+        for ((resource, acls) <- resourceToAcl) {
+          println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          val aclBindings = acls.map(acl => new AclBinding(resource, acl)).asJavaCollection
+          adminClient.createAcls(aclBindings).all().get()
+        }
+
+        listAcls()
+      }
+    }
+
+    def removeAcls(): Unit = {
+      withAdminClient(opts) { adminClient =>
+        val filterToAcl = getResourceFilterToAcls(opts)
+
+        for ((filter, acls) <- filterToAcl) {
+          if (acls.isEmpty) {
+            if (confirmAction(opts, s"Are you sure you want to delete all ACLs for resource filter `$filter`? (y/n)"))
+              removeAcls(adminClient, acls, filter)
+          } else {
+            if (confirmAction(opts, s"Are you sure you want to remove ACLs: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline from resource filter `$filter`? (y/n)"))
+              removeAcls(adminClient, acls, filter)
+          }
+        }
+
+        listAcls()
+      }
+    }
+
+    def listAcls(): Unit = {
+      withAdminClient(opts) { adminClient =>
+        val filters = getResourceFilter(opts, dieIfNoResourceFound = false)
+        val listPrincipals = getPrincipals(opts, opts.listPrincipalsOpt)
+        val resourceToAcls = getAcls(adminClient, filters)
+
+        if (listPrincipals.isEmpty) {
+          for ((resource, acls) <- resourceToAcls)
+            println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+        } else {
+          listPrincipals.foreach(principal => {
+            println(s"ACLs for principal `$principal`")
+            val filteredResourceToAcls =  resourceToAcls.mapValues(acls =>
+              acls.filter(acl => principal.toString.equals(acl.principal))).filter(entry => entry._2.nonEmpty)
+
+            for ((resource, acls) <- filteredResourceToAcls)
+              println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          })
+        }
+      }
+    }
+
+    private def removeAcls(adminClient: Admin, acls: Set[AccessControlEntry], filter: ResourcePatternFilter): Unit = {
+      if (acls.isEmpty)
+        adminClient.deleteAcls(List(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asJava).all().get()
+      else {
+        val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, acl.toFilter)).toList.asJava
+        adminClient.deleteAcls(aclBindingFilters).all().get()
+      }
+    }
+
+    private def getAcls(adminClient: Admin, filters: Set[ResourcePatternFilter]): Map[ResourcePattern, Set[AccessControlEntry]] = {
+      val aclBindings =
+        if (filters.isEmpty) adminClient.describeAcls(AclBindingFilter.ANY).values().get().asScala.toList
+        else {
+          val results = for (filter <- filters) yield {
+            adminClient.describeAcls(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).values().get().asScala.toList
+          }
+          results.reduceLeft(_ ++ _)
+        }
+
+      val resourceToAcls = mutable.Map[ResourcePattern, Set[AccessControlEntry]]().withDefaultValue(Set())
+
+      aclBindings.foreach(aclBinding => resourceToAcls(aclBinding.pattern()) = resourceToAcls(aclBinding.pattern()) + aclBinding.entry())
+      resourceToAcls.toMap
+    }
+  }
+
+  class AuthorizerService(val authorizerClassName: String, val opts: AclCommandOptions) extends AclCommandService with Logging {
+
+    private def withAuthorizer()(f: Authorizer => Unit): Unit = {
+      // It is possible that zookeeper.set.acl could be true without SASL if mutual certificate authentication is configured.
+      // We will default the value of zookeeper.set.acl to true or false based on whether SASL is configured,
+      // but if SASL is not configured and zookeeper.set.acl is supposed to be true due to mutual certificate authentication
+      // then it will be up to the user to explicitly specify zookeeper.set.acl=true in the authorizer-properties.
+      val defaultProps = Map(KafkaConfig.ZkEnableSecureAclsProp -> JaasUtils.isZkSaslEnabled)
+      val authorizerPropertiesWithoutTls =
+        if (opts.options.has(opts.authorizerPropertiesOpt)) {
+          val authorizerProperties = opts.options.valuesOf(opts.authorizerPropertiesOpt).asScala
+          defaultProps ++ CommandLineUtils.parseKeyValueArgs(authorizerProperties, acceptMissingValue = false).asScala
+        } else {
+          defaultProps
+        }
+      val authorizerProperties =
+        if (opts.options.has(opts.zkTlsConfigFile)) {
+          // load in TLS configs both with and without the "authorizer." prefix
+          val validKeys = (KafkaConfig.ZkSslConfigToSystemPropertyMap.keys.toList ++ KafkaConfig.ZkSslConfigToSystemPropertyMap.keys.map("authorizer." + _).toList).asJava
+          authorizerPropertiesWithoutTls ++ Utils.loadProps(opts.options.valueOf(opts.zkTlsConfigFile), validKeys).asInstanceOf[java.util.Map[String, Any]].asScala
+        }
+        else
+          authorizerPropertiesWithoutTls
+
+      val authZ = AuthorizerUtils.createAuthorizer(authorizerClassName)
+      try {
+        authZ.configure(authorizerProperties.asJava)
+        f(authZ)
+      }
+      finally CoreUtils.swallow(authZ.close(), this)
+    }
+
+    def addAcls(): Unit = {
+      val resourceToAcl = getResourceToAcls(opts)
+      withAuthorizer() { authorizer =>
+        for ((resource, acls) <- resourceToAcl) {
+          println(s"Adding ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          val aclBindings = acls.map(acl => new AclBinding(resource, acl))
+          authorizer.createAcls(null,aclBindings.toList.asJava).asScala.map(_.toCompletableFuture.get).foreach { result =>
+            result.exception.asScala.foreach { exception =>
+              println(s"Error while adding ACLs: ${exception.getMessage}")
+              println(Utils.stackTrace(exception))
+            }
+          }
+        }
+
+        listAcls()
+      }
+    }
+
+    def removeAcls(): Unit = {
+      withAuthorizer() { authorizer =>
+        val filterToAcl = getResourceFilterToAcls(opts)
+
+        for ((filter, acls) <- filterToAcl) {
+          if (acls.isEmpty) {
+            if (confirmAction(opts, s"Are you sure you want to delete all ACLs for resource filter `$filter`? (y/n)"))
+              removeAcls(authorizer, acls, filter)
+          } else {
+            if (confirmAction(opts, s"Are you sure you want to remove ACLs: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline from resource filter `$filter`? (y/n)"))
+              removeAcls(authorizer, acls, filter)
+          }
+        }
+
+        listAcls()
+      }
+    }
+
+    def listAcls(): Unit = {
+      withAuthorizer() { authorizer =>
+        val filters = getResourceFilter(opts, dieIfNoResourceFound = false)
+        val listPrincipals = getPrincipals(opts, opts.listPrincipalsOpt)
+        val resourceToAcls = getAcls(authorizer, filters)
+
+        if (listPrincipals.isEmpty) {
+          for ((resource, acls) <- resourceToAcls)
+            println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+        } else {
+          listPrincipals.foreach(principal => {
+            println(s"ACLs for principal `$principal`")
+            val filteredResourceToAcls =  resourceToAcls.mapValues(acls =>
+              acls.filter(acl => principal.toString.equals(acl.principal))).filter(entry => entry._2.nonEmpty)
+
+            for ((resource, acls) <- filteredResourceToAcls)
+              println(s"Current ACLs for resource `$resource`: $Newline ${acls.map("\t" + _).mkString(Newline)} $Newline")
+          })
+        }
+      }
+    }
+
+    private def removeAcls(authorizer: Authorizer, acls: Set[AccessControlEntry], filter: ResourcePatternFilter): Unit = {
+      val result = if (acls.isEmpty)
+        authorizer.deleteAcls(null, List(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asJava)
+      else {
+        val aclBindingFilters = acls.map(acl => new AclBindingFilter(filter, acl.toFilter)).toList.asJava
+        authorizer.deleteAcls(null, aclBindingFilters)
+      }
+      result.asScala.map(_.toCompletableFuture.get).foreach { result =>
+        result.exception.asScala.foreach { exception =>
+          println(s"Error while removing ACLs: ${exception.getMessage}")
+          println(Utils.stackTrace(exception))
+        }
+        result.aclBindingDeleteResults.asScala.foreach { deleteResult =>
+          deleteResult.exception.asScala.foreach { exception =>
+            println(s"Error while removing ACLs: ${exception.getMessage}")
+            println(Utils.stackTrace(exception))
+          }
+        }
+      }
+    }
+
+    private def getAcls(authorizer: Authorizer, filters: Set[ResourcePatternFilter]): Map[ResourcePattern, Set[AccessControlEntry]] = {
+      val aclBindings =
+        if (filters.isEmpty) authorizer.acls(AclBindingFilter.ANY).asScala
+        else {
+          val results = for (filter <- filters) yield {
+            authorizer.acls(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).asScala
+          }
+          results.reduceLeft(_ ++ _)
+        }
+
+      val resourceToAcls = mutable.Map[ResourcePattern, Set[AccessControlEntry]]().withDefaultValue(Set())
+
+      aclBindings.foreach(aclBinding => resourceToAcls(aclBinding.pattern()) = resourceToAcls(aclBinding.pattern()) + aclBinding.entry())
+      resourceToAcls.toMap
+    }
+  }
+
+  private def getResourceToAcls(opts: AclCommandOptions): Map[ResourcePattern, Set[AccessControlEntry]] = {
+    val patternType: PatternType = opts.options.valueOf(opts.resourcePatternType)
+    if (!patternType.isSpecific)
+      CommandLineUtils.printUsageAndDie(opts.parser, s"A '--resource-pattern-type' value of '$patternType' is not valid when adding acls.")
+
+    val resourceToAcl = getResourceFilterToAcls(opts).map {
+      case (filter, acls) =>
+        new ResourcePattern(filter.resourceType(), filter.name(), filter.patternType()) -> acls
+    }
+
+    if (resourceToAcl.values.exists(_.isEmpty))
+      CommandLineUtils.printUsageAndDie(opts.parser, "You must specify one of: --allow-principal, --deny-principal when trying to add ACLs.")
+
+    resourceToAcl
+  }
+
+  private def getResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
+    var resourceToAcls = Map.empty[ResourcePatternFilter, Set[AccessControlEntry]]
+
+    //if none of the --producer or --consumer options are specified , just construct ACLs from CLI options.
+    if (!opts.options.has(opts.producerOpt) && !opts.options.has(opts.consumerOpt)) {
+      resourceToAcls ++= getCliResourceFilterToAcls(opts)
+    }
+
+    //users are allowed to specify both --producer and --consumer options in a single command.
+    if (opts.options.has(opts.producerOpt))
+      resourceToAcls ++= getProducerResourceFilterToAcls(opts)
+
+    if (opts.options.has(opts.consumerOpt))
+      resourceToAcls ++= getConsumerResourceFilterToAcls(opts).map { case (k, v) => k -> (v ++ resourceToAcls.getOrElse(k, Set.empty[AccessControlEntry])) }
+
+    validateOperation(opts, resourceToAcls)
+
+    resourceToAcls
+  }
+
+  private def getProducerResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
+    val filters = getResourceFilter(opts)
+
+    val topics: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.TOPIC)
+    val transactionalIds: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.TRANSACTIONAL_ID)
+    val enableIdempotence = opts.options.has(opts.idempotentOpt)
+
+    val topicAcls = getAcl(opts, Set(WRITE, DESCRIBE, CREATE))
+    val transactionalIdAcls = getAcl(opts, Set(WRITE, DESCRIBE))
+
+    //Write, Describe, Create permission on topics, Write, Describe on transactionalIds
+    topics.map(_ -> topicAcls).toMap ++
+      transactionalIds.map(_ -> transactionalIdAcls).toMap ++
+        (if (enableIdempotence)
+          Map(ClusterResourceFilter -> getAcl(opts, Set(IDEMPOTENT_WRITE)))
+        else
+          Map.empty)
+  }
+
+  private def getConsumerResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
+    val filters = getResourceFilter(opts)
+
+    val topics: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.TOPIC)
+    val groups: Set[ResourcePatternFilter] = filters.filter(_.resourceType == JResourceType.GROUP)
+
+    //Read, Describe on topic, Read on consumerGroup
+
+    val acls = getAcl(opts, Set(READ, DESCRIBE))
+
+    topics.map(_ -> acls).toMap[ResourcePatternFilter, Set[AccessControlEntry]] ++
+      groups.map(_ -> getAcl(opts, Set(READ))).toMap[ResourcePatternFilter, Set[AccessControlEntry]]
+  }
+
+  private def getCliResourceFilterToAcls(opts: AclCommandOptions): Map[ResourcePatternFilter, Set[AccessControlEntry]] = {
+    val acls = getAcl(opts)
+    val filters = getResourceFilter(opts)
+    filters.map(_ -> acls).toMap
+  }
+
+  private def getAcl(opts: AclCommandOptions, operations: Set[AclOperation]): Set[AccessControlEntry] = {
+    val allowedPrincipals = getPrincipals(opts, opts.allowPrincipalsOpt)
+
+    val deniedPrincipals = getPrincipals(opts, opts.denyPrincipalsOpt)
+
+    val allowedHosts = getHosts(opts, opts.allowHostsOpt, opts.allowPrincipalsOpt)
+
+    val deniedHosts = getHosts(opts, opts.denyHostsOpt, opts.denyPrincipalsOpt)
+
+    val acls = new collection.mutable.HashSet[AccessControlEntry]
+    if (allowedHosts.nonEmpty && allowedPrincipals.nonEmpty)
+      acls ++= getAcls(allowedPrincipals, ALLOW, operations, allowedHosts)
+
+    if (deniedHosts.nonEmpty && deniedPrincipals.nonEmpty)
+      acls ++= getAcls(deniedPrincipals, DENY, operations, deniedHosts)
+
+    acls.toSet
+  }
+
+  private def getAcl(opts: AclCommandOptions): Set[AccessControlEntry] = {
+    val operations = opts.options.valuesOf(opts.operationsOpt).asScala
+      .map(operation => JSecurityUtils.operation(operation.trim)).toSet
+    getAcl(opts, operations)
+  }
+
+  def getAcls(principals: Set[KafkaPrincipal], permissionType: AclPermissionType, operations: Set[AclOperation],
+              hosts: Set[String]): Set[AccessControlEntry] = {
+    for {
+      principal <- principals
+      operation <- operations
+      host <- hosts
+    } yield new AccessControlEntry(principal.toString, host, operation, permissionType)
+  }
+
+  private def getHosts(opts: AclCommandOptions, hostOptionSpec: ArgumentAcceptingOptionSpec[String],
+                       principalOptionSpec: ArgumentAcceptingOptionSpec[String]): Set[String] = {
+    if (opts.options.has(hostOptionSpec))
+      opts.options.valuesOf(hostOptionSpec).asScala.map(_.trim).toSet
+    else if (opts.options.has(principalOptionSpec))
+      Set[String](AclEntry.WildcardHost)
+    else
+      Set.empty[String]
+  }
+
+  private def getPrincipals(opts: AclCommandOptions, principalOptionSpec: ArgumentAcceptingOptionSpec[String]): Set[KafkaPrincipal] = {
+    if (opts.options.has(principalOptionSpec))
+      opts.options.valuesOf(principalOptionSpec).asScala.map(s => JSecurityUtils.parseKafkaPrincipal(s.trim)).toSet
+    else
+      Set.empty[KafkaPrincipal]
+  }
+
+  private def getResourceFilter(opts: AclCommandOptions, dieIfNoResourceFound: Boolean = true): Set[ResourcePatternFilter] = {
+    val patternType: PatternType = opts.options.valueOf(opts.resourcePatternType)
+
+    var resourceFilters = Set.empty[ResourcePatternFilter]
+    if (opts.options.has(opts.topicOpt))
+      opts.options.valuesOf(opts.topicOpt).asScala.foreach(topic => resourceFilters += new ResourcePatternFilter(JResourceType.TOPIC, topic.trim, patternType))
+
+    if (patternType == PatternType.LITERAL && (opts.options.has(opts.clusterOpt) || opts.options.has(opts.idempotentOpt)))
+      resourceFilters += ClusterResourceFilter
+
+    if (opts.options.has(opts.groupOpt))
+      opts.options.valuesOf(opts.groupOpt).asScala.foreach(group => resourceFilters += new ResourcePatternFilter(JResourceType.GROUP, group.trim, patternType))
+
+    if (opts.options.has(opts.transactionalIdOpt))
+      opts.options.valuesOf(opts.transactionalIdOpt).asScala.foreach(transactionalId =>
+        resourceFilters += new ResourcePatternFilter(JResourceType.TRANSACTIONAL_ID, transactionalId, patternType))
+
+    if (opts.options.has(opts.delegationTokenOpt))
+      opts.options.valuesOf(opts.delegationTokenOpt).asScala.foreach(token => resourceFilters += new ResourcePatternFilter(JResourceType.DELEGATION_TOKEN, token.trim, patternType))
+
+    if (resourceFilters.isEmpty && dieIfNoResourceFound)
+      CommandLineUtils.printUsageAndDie(opts.parser, "You must provide at least one resource: --topic <topic> or --cluster or --group <group> or --delegation-token <Delegation Token ID>")
+
+    resourceFilters
+  }
+
+  private def confirmAction(opts: AclCommandOptions, msg: String): Boolean = {
+    if (opts.options.has(opts.forceOpt))
+        return true
+    println(msg)
+    StdIn.readLine().equalsIgnoreCase("y")
+  }
+
+  private def validateOperation(opts: AclCommandOptions, resourceToAcls: Map[ResourcePatternFilter, Set[AccessControlEntry]]): Unit = {
+    for ((resource, acls) <- resourceToAcls) {
+      val validOps = AclEntry.supportedOperations(resource.resourceType) + AclOperation.ALL
+      if ((acls.map(_.operation) -- validOps).nonEmpty)
+        CommandLineUtils.printUsageAndDie(opts.parser, s"ResourceType ${resource.resourceType} only supports operations ${validOps.mkString(",")}")
+    }
+  }
+
+  class AclCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    val CommandConfigDoc = "A property file containing configs to be passed to Admin Client."
+
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", "A list of host/port pairs to use for establishing the connection to the Kafka cluster." +
+      " This list should be in the form host1:port1,host2:port2,... This config is required for acl management using admin client API.")
+      .withRequiredArg
+      .describedAs("server to connect to")
+      .ofType(classOf[String])
+
+    val commandConfigOpt = parser.accepts("command-config", CommandConfigDoc)
+      .withOptionalArg()
+      .describedAs("command-config")
+      .ofType(classOf[String])
+
+    val authorizerOpt = parser.accepts("authorizer", "Fully qualified class name of the authorizer, defaults to kafka.security.authorizer.AclAuthorizer.")
+      .withRequiredArg
+      .describedAs("authorizer")
+      .ofType(classOf[String])
+
+    val authorizerPropertiesOpt = parser.accepts("authorizer-properties", "REQUIRED: properties required to configure an instance of Authorizer. " +
+      "These are key=val pairs. For the default authorizer the example values are: zookeeper.connect=localhost:2181")
+      .withRequiredArg
+      .describedAs("authorizer-properties")
+      .ofType(classOf[String])
+
+    val topicOpt = parser.accepts("topic", "topic to which ACLs should be added or removed. " +
+      "A value of * indicates ACL should apply to all topics.")
+      .withRequiredArg
+      .describedAs("topic")
+      .ofType(classOf[String])
+
+    val clusterOpt = parser.accepts("cluster", "Add/Remove cluster ACLs.")
+    val groupOpt = parser.accepts("group", "Consumer Group to which the ACLs should be added or removed. " +
+      "A value of * indicates the ACLs should apply to all groups.")
+      .withRequiredArg
+      .describedAs("group")
+      .ofType(classOf[String])
+
+    val transactionalIdOpt = parser.accepts("transactional-id", "The transactionalId to which ACLs should " +
+      "be added or removed. A value of * indicates the ACLs should apply to all transactionalIds.")
+      .withRequiredArg
+      .describedAs("transactional-id")
+      .ofType(classOf[String])
+
+    val idempotentOpt = parser.accepts("idempotent", "Enable idempotence for the producer. This should be " +
+      "used in combination with the --producer option. Note that idempotence is enabled automatically if " +
+      "the producer is authorized to a particular transactional-id.")
+
+    val delegationTokenOpt = parser.accepts("delegation-token", "Delegation token to which ACLs should be added or removed. " +
+      "A value of * indicates ACL should apply to all tokens.")
+      .withRequiredArg
+      .describedAs("delegation-token")
+      .ofType(classOf[String])
+
+    val resourcePatternType = parser.accepts("resource-pattern-type", "The type of the resource pattern or pattern filter. " +
+      "When adding acls, this should be a specific pattern type, e.g. 'literal' or 'prefixed'. " +
+      "When listing or removing acls, a specific pattern type can be used to list or remove acls from specific resource patterns, " +
+      "or use the filter values of 'any' or 'match', where 'any' will match any pattern type, but will match the resource name exactly, " +
+      "where as 'match' will perform pattern matching to list or remove all acls that affect the supplied resource(s). " +
+      "WARNING: 'match', when used in combination with the '--remove' switch, should be used with care.")
+      .withRequiredArg()
+      .ofType(classOf[String])
+      .withValuesConvertedBy(new PatternTypeConverter())
+      .defaultsTo(PatternType.LITERAL)
+
+    val addOpt = parser.accepts("add", "Indicates you are trying to add ACLs.")
+    val removeOpt = parser.accepts("remove", "Indicates you are trying to remove ACLs.")
+    val listOpt = parser.accepts("list", "List ACLs for the specified resource, use --topic <topic> or --group <group> or --cluster to specify a resource.")
+
+    val operationsOpt = parser.accepts("operation", "Operation that is being allowed or denied. Valid operation names are: " + Newline +
+      AclEntry.AclOperations.map("\t" + JSecurityUtils.operationName(_)).mkString(Newline) + Newline)
+      .withRequiredArg
+      .ofType(classOf[String])
+      .defaultsTo(JSecurityUtils.operationName(AclOperation.ALL))
+
+    val allowPrincipalsOpt = parser.accepts("allow-principal", "principal is in principalType:name format." +
+      " Note that principalType must be supported by the Authorizer being used." +
+      " For example, User:* is the wild card indicating all users.")
+      .withRequiredArg
+      .describedAs("allow-principal")
+      .ofType(classOf[String])
+
+    val denyPrincipalsOpt = parser.accepts("deny-principal", "principal is in principalType:name format. " +
+      "By default anyone not added through --allow-principal is denied access. " +
+      "You only need to use this option as negation to already allowed set. " +
+      "Note that principalType must be supported by the Authorizer being used. " +
+      "For example if you wanted to allow access to all users in the system but not test-user you can define an ACL that " +
+      "allows access to User:* and specify --deny-principal=User:test@EXAMPLE.COM. " +
+      "AND PLEASE REMEMBER DENY RULES TAKES PRECEDENCE OVER ALLOW RULES.")
+      .withRequiredArg
+      .describedAs("deny-principal")
+      .ofType(classOf[String])
+
+    val listPrincipalsOpt = parser.accepts("principal", "List ACLs for the specified principal. principal is in principalType:name format." +
+      " Note that principalType must be supported by the Authorizer being used. Multiple --principal option can be passed.")
+      .withOptionalArg()
+      .describedAs("principal")
+      .ofType(classOf[String])
+
+    val allowHostsOpt = parser.accepts("allow-host", "Host from which principals listed in --allow-principal will have access. " +
+      "If you have specified --allow-principal then the default for this option will be set to * which allows access from all hosts.")
+      .withRequiredArg
+      .describedAs("allow-host")
+      .ofType(classOf[String])
+
+    val denyHostsOpt = parser.accepts("deny-host", "Host from which principals listed in --deny-principal will be denied access. " +
+      "If you have specified --deny-principal then the default for this option will be set to * which denies access from all hosts.")
+      .withRequiredArg
+      .describedAs("deny-host")
+      .ofType(classOf[String])
+
+    val producerOpt = parser.accepts("producer", "Convenience option to add/remove ACLs for producer role. " +
+      "This will generate ACLs that allows WRITE,DESCRIBE and CREATE on topic.")
+
+    val consumerOpt = parser.accepts("consumer", "Convenience option to add/remove ACLs for consumer role. " +
+      "This will generate ACLs that allows READ,DESCRIBE on topic and READ on group.")
+
+    val forceOpt = parser.accepts("force", "Assume Yes to all queries and do not prompt.")
+
+    val zkTlsConfigFile = parser.accepts("zk-tls-config-file",
+      "Identifies the file where ZooKeeper client TLS connectivity properties for the authorizer are defined.  Any properties other than the following (with or without an \"authorizer.\" prefix) are ignored: " +
+        KafkaConfig.ZkSslConfigToSystemPropertyMap.keys.toList.sorted.mkString(", ") +
+        ". Note that if SASL is not configured and zookeeper.set.acl is supposed to be true due to mutual certificate authentication being used" +
+        " then it is necessary to explicitly specify --authorizer-properties zookeeper.set.acl=true")
+      .withRequiredArg().describedAs("Authorizer ZooKeeper TLS configuration").ofType(classOf[String])
+
+    options = parser.parse(args: _*)
+
+    def checkArgs(): Unit = {
+      if (options.has(bootstrapServerOpt) && options.has(authorizerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "Only one of --bootstrap-server or --authorizer must be specified")
+
+      if (!options.has(bootstrapServerOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, authorizerPropertiesOpt)
+
+      if (options.has(commandConfigOpt) && !options.has(bootstrapServerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "The --command-config option can only be used with --bootstrap-server option")
+
+      if (options.has(authorizerPropertiesOpt) && options.has(bootstrapServerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "The --authorizer-properties option can only be used with --authorizer option")
+
+      val actions = Seq(addOpt, removeOpt, listOpt).count(options.has)
+      if (actions != 1)
+        CommandLineUtils.printUsageAndDie(parser, "Command must include exactly one action: --list, --add, --remove. ")
+
+      CommandLineUtils.checkInvalidArgs(parser, options, listOpt, Set(producerOpt, consumerOpt, allowHostsOpt, allowPrincipalsOpt, denyHostsOpt, denyPrincipalsOpt))
+
+      //when --producer or --consumer is specified , user should not specify operations as they are inferred and we also disallow --deny-principals and --deny-hosts.
+      CommandLineUtils.checkInvalidArgs(parser, options, producerOpt, Set(operationsOpt, denyPrincipalsOpt, denyHostsOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, consumerOpt, Set(operationsOpt, denyPrincipalsOpt, denyHostsOpt))
+
+      if (options.has(listPrincipalsOpt) && !options.has(listOpt))
+        CommandLineUtils.printUsageAndDie(parser, "The --principal option is only available if --list is set")
+
+      if (options.has(producerOpt) && !options.has(topicOpt))
+        CommandLineUtils.printUsageAndDie(parser, "With --producer you must specify a --topic")
+
+      if (options.has(idempotentOpt) && !options.has(producerOpt))
+        CommandLineUtils.printUsageAndDie(parser, "The --idempotent option is only available if --producer is set")
+
+      if (options.has(consumerOpt) && (!options.has(topicOpt) || !options.has(groupOpt) || (!options.has(producerOpt) && (options.has(clusterOpt) || options.has(transactionalIdOpt)))))
+        CommandLineUtils.printUsageAndDie(parser, "With --consumer you must specify a --topic and a --group and no --cluster or --transactional-id option should be specified.")
+    }
+  }
+}
+
+class PatternTypeConverter extends EnumConverter[PatternType](classOf[PatternType]) {
+
+  override def convert(value: String): PatternType = {
+    val patternType = super.convert(value)
+    if (patternType.isUnknown)
+      throw new ValueConversionException("Unknown resource-pattern-type: " + value)
+
+    patternType
+  }
+
+  override def valuePattern: String = PatternType.values
+    .filter(_ != PatternType.UNKNOWN)
+    .mkString("|")
+}

core/src/main/scala/kafka/admin/AdminOperationException.scala

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+class AdminOperationException(val error: String, cause: Throwable) extends RuntimeException(error, cause) {
+  def this(error: Throwable) = this(error.getMessage, error)
+  def this(msg: String) = this(msg, null)
+}

core/src/main/scala/kafka/admin/AdminUtils.scala

@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.util.Random
+
+import kafka.utils.Logging
+import org.apache.kafka.common.errors.{InvalidPartitionsException, InvalidReplicationFactorException}
+
+import collection.{Map, mutable, _}
+
+object AdminUtils extends Logging {
+  val rand = new Random
+  val AdminClientId = "__admin_client"
+
+  /**
+   * There are 3 goals of replica assignment:
+   *
+   * <ol>
+   * <li> Spread the replicas evenly among brokers.</li>
+   * <li> For partitions assigned to a particular broker, their other replicas are spread over the other brokers.</li>
+   * <li> If all brokers have rack information, assign the replicas for each partition to different racks if possible</li>
+   * </ol>
+   *
+   * To achieve this goal for replica assignment without considering racks, we:
+   * <ol>
+   * <li> Assign the first replica of each partition by round-robin, starting from a random position in the broker list.</li>
+   * <li> Assign the remaining replicas of each partition with an increasing shift.</li>
+   * </ol>
+   *
+   * Here is an example of assigning
+   * <table cellpadding="2" cellspacing="2">
+   * <tr><th>broker-0</th><th>broker-1</th><th>broker-2</th><th>broker-3</th><th>broker-4</th><th>&nbsp;</th></tr>
+   * <tr><td>p0      </td><td>p1      </td><td>p2      </td><td>p3      </td><td>p4      </td><td>(1st replica)</td></tr>
+   * <tr><td>p5      </td><td>p6      </td><td>p7      </td><td>p8      </td><td>p9      </td><td>(1st replica)</td></tr>
+   * <tr><td>p4      </td><td>p0      </td><td>p1      </td><td>p2      </td><td>p3      </td><td>(2nd replica)</td></tr>
+   * <tr><td>p8      </td><td>p9      </td><td>p5      </td><td>p6      </td><td>p7      </td><td>(2nd replica)</td></tr>
+   * <tr><td>p3      </td><td>p4      </td><td>p0      </td><td>p1      </td><td>p2      </td><td>(3nd replica)</td></tr>
+   * <tr><td>p7      </td><td>p8      </td><td>p9      </td><td>p5      </td><td>p6      </td><td>(3nd replica)</td></tr>
+   * </table>
+   *
+   * <p>
+   * To create rack aware assignment, this API will first create a rack alternated broker list. For example,
+   * from this brokerID -> rack mapping:</p>
+   * 0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1"
+   * <br><br>
+   * <p>
+   * The rack alternated list will be:
+   * </p>
+   * 0, 3, 1, 5, 4, 2
+   * <br><br>
+   * <p>
+   * Then an easy round-robin assignment can be applied. Assume 6 partitions with replication factor of 3, the assignment
+   * will be:
+   * </p>
+   * 0 -> 0,3,1 <br>
+   * 1 -> 3,1,5 <br>
+   * 2 -> 1,5,4 <br>
+   * 3 -> 5,4,2 <br>
+   * 4 -> 4,2,0 <br>
+   * 5 -> 2,0,3 <br>
+   * <br>
+   * <p>
+   * Once it has completed the first round-robin, if there are more partitions to assign, the algorithm will start
+   * shifting the followers. This is to ensure we will not always get the same set of sequences.
+   * In this case, if there is another partition to assign (partition #6), the assignment will be:
+   * </p>
+   * 6 -> 0,4,2 (instead of repeating 0,3,1 as partition 0)
+   * <br><br>
+   * <p>
+   * The rack aware assignment always chooses the 1st replica of the partition using round robin on the rack alternated
+   * broker list. For rest of the replicas, it will be biased towards brokers on racks that do not have
+   * any replica assignment, until every rack has a replica. Then the assignment will go back to round-robin on
+   * the broker list.
+   * </p>
+   * <br>
+   * <p>
+   * As the result, if the number of replicas is equal to or greater than the number of racks, it will ensure that
+   * each rack will get at least one replica. Otherwise, each rack will get at most one replica. In a perfect
+   * situation where the number of replicas is the same as the number of racks and each rack has the same number of
+   * brokers, it guarantees that the replica distribution is even across brokers and racks.
+   * </p>
+   * @return a Map from partition id to replica ids
+   * @throws AdminOperationException If rack information is supplied but it is incomplete, or if it is not possible to
+   *                                 assign each replica to a unique rack.
+   *
+   */
+  def assignReplicasToBrokers(brokerMetadatas: Seq[BrokerMetadata],
+                              nPartitions: Int,
+                              replicationFactor: Int,
+                              fixedStartIndex: Int = -1,
+                              startPartitionId: Int = -1): Map[Int, Seq[Int]] = {
+    if (nPartitions <= 0)
+      throw new InvalidPartitionsException("Number of partitions must be larger than 0.")
+    if (replicationFactor <= 0)
+      throw new InvalidReplicationFactorException("Replication factor must be larger than 0.")
+    if (replicationFactor > brokerMetadatas.size)
+      throw new InvalidReplicationFactorException(s"Replication factor: $replicationFactor larger than available brokers: ${brokerMetadatas.size}.")
+    if (brokerMetadatas.forall(_.rack.isEmpty))
+      assignReplicasToBrokersRackUnaware(nPartitions, replicationFactor, brokerMetadatas.map(_.id), fixedStartIndex,
+        startPartitionId)
+    else {
+      if (brokerMetadatas.exists(_.rack.isEmpty))
+        throw new AdminOperationException("Not all brokers have rack information for replica rack aware assignment.")
+      assignReplicasToBrokersRackAware(nPartitions, replicationFactor, brokerMetadatas, fixedStartIndex,
+        startPartitionId)
+    }
+  }
+
+  private def assignReplicasToBrokersRackUnaware(nPartitions: Int,
+                                                 replicationFactor: Int,
+                                                 brokerList: Seq[Int],
+                                                 fixedStartIndex: Int,
+                                                 startPartitionId: Int): Map[Int, Seq[Int]] = {
+    val ret = mutable.Map[Int, Seq[Int]]()
+    val brokerArray = brokerList.toArray
+    val startIndex = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(brokerArray.length)
+    var currentPartitionId = math.max(0, startPartitionId)
+    var nextReplicaShift = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(brokerArray.length)
+    for (_ <- 0 until nPartitions) {
+      if (currentPartitionId > 0 && (currentPartitionId % brokerArray.length == 0))
+        nextReplicaShift += 1
+      val firstReplicaIndex = (currentPartitionId + startIndex) % brokerArray.length
+      val replicaBuffer = mutable.ArrayBuffer(brokerArray(firstReplicaIndex))
+      for (j <- 0 until replicationFactor - 1)
+        replicaBuffer += brokerArray(replicaIndex(firstReplicaIndex, nextReplicaShift, j, brokerArray.length))
+      ret.put(currentPartitionId, replicaBuffer)
+      currentPartitionId += 1
+    }
+    ret
+  }
+
+  private def assignReplicasToBrokersRackAware(nPartitions: Int,
+                                               replicationFactor: Int,
+                                               brokerMetadatas: Seq[BrokerMetadata],
+                                               fixedStartIndex: Int,
+                                               startPartitionId: Int): Map[Int, Seq[Int]] = {
+    val brokerRackMap = brokerMetadatas.collect { case BrokerMetadata(id, Some(rack)) =>
+      id -> rack
+    }.toMap
+    val numRacks = brokerRackMap.values.toSet.size
+    val arrangedBrokerList = getRackAlternatedBrokerList(brokerRackMap)
+    val numBrokers = arrangedBrokerList.size
+    val ret = mutable.Map[Int, Seq[Int]]()
+    val startIndex = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(arrangedBrokerList.size)
+    var currentPartitionId = math.max(0, startPartitionId)
+    var nextReplicaShift = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(arrangedBrokerList.size)
+    for (_ <- 0 until nPartitions) {
+      if (currentPartitionId > 0 && (currentPartitionId % arrangedBrokerList.size == 0))
+        nextReplicaShift += 1
+      val firstReplicaIndex = (currentPartitionId + startIndex) % arrangedBrokerList.size
+      val leader = arrangedBrokerList(firstReplicaIndex)
+      val replicaBuffer = mutable.ArrayBuffer(leader)
+      val racksWithReplicas = mutable.Set(brokerRackMap(leader))
+      val brokersWithReplicas = mutable.Set(leader)
+      var k = 0
+      for (_ <- 0 until replicationFactor - 1) {
+        var done = false
+        while (!done) {
+          val broker = arrangedBrokerList(replicaIndex(firstReplicaIndex, nextReplicaShift * numRacks, k, arrangedBrokerList.size))
+          val rack = brokerRackMap(broker)
+          // Skip this broker if
+          // 1. there is already a broker in the same rack that has assigned a replica AND there is one or more racks
+          //    that do not have any replica, or
+          // 2. the broker has already assigned a replica AND there is one or more brokers that do not have replica assigned
+          if ((!racksWithReplicas.contains(rack) || racksWithReplicas.size == numRacks)
+              && (!brokersWithReplicas.contains(broker) || brokersWithReplicas.size == numBrokers)) {
+            replicaBuffer += broker
+            racksWithReplicas += rack
+            brokersWithReplicas += broker
+            done = true
+          }
+          k += 1
+        }
+      }
+      ret.put(currentPartitionId, replicaBuffer)
+      currentPartitionId += 1
+    }
+    ret
+  }
+
+  /**
+    * Given broker and rack information, returns a list of brokers alternated by the rack. Assume
+    * this is the rack and its brokers:
+    *
+    * rack1: 0, 1, 2
+    * rack2: 3, 4, 5
+    * rack3: 6, 7, 8
+    *
+    * This API would return the list of 0, 3, 6, 1, 4, 7, 2, 5, 8
+    *
+    * This is essential to make sure that the assignReplicasToBrokers API can use such list and
+    * assign replicas to brokers in a simple round-robin fashion, while ensuring an even
+    * distribution of leader and replica counts on each broker and that replicas are
+    * distributed to all racks.
+    */
+  private[admin] def getRackAlternatedBrokerList(brokerRackMap: Map[Int, String]): IndexedSeq[Int] = {
+    val brokersIteratorByRack = getInverseMap(brokerRackMap).map { case (rack, brokers) =>
+      (rack, brokers.iterator)
+    }
+    val racks = brokersIteratorByRack.keys.toArray.sorted
+    val result = new mutable.ArrayBuffer[Int]
+    var rackIndex = 0
+    while (result.size < brokerRackMap.size) {
+      val rackIterator = brokersIteratorByRack(racks(rackIndex))
+      if (rackIterator.hasNext)
+        result += rackIterator.next()
+      rackIndex = (rackIndex + 1) % racks.length
+    }
+    result
+  }
+
+  private[admin] def getInverseMap(brokerRackMap: Map[Int, String]): Map[String, Seq[Int]] = {
+    brokerRackMap.toSeq.map { case (id, rack) => (rack, id) }
+      .groupBy { case (rack, _) => rack }
+      .map { case (rack, rackAndIdList) => (rack, rackAndIdList.map { case (_, id) => id }.sorted) }
+  }
+
+  private def replicaIndex(firstReplicaIndex: Int, secondReplicaShift: Int, replicaIndex: Int, nBrokers: Int): Int = {
+    val shift = 1 + (secondReplicaShift + replicaIndex) % (nBrokers - 1)
+    (firstReplicaIndex + shift) % nBrokers
+  }
+
+}

core/src/main/scala/kafka/admin/BrokerApiVersionsCommand.scala

@@ -0,0 +1,323 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.io.PrintStream
+import java.io.IOException
+import java.util.Properties
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.{ConcurrentLinkedQueue, TimeUnit}
+
+import kafka.utils.{CommandDefaultOptions, CommandLineUtils}
+import kafka.utils.Logging
+import org.apache.kafka.common.utils.Utils
+import org.apache.kafka.clients.{ApiVersions, ClientDnsLookup, ClientResponse, ClientUtils, CommonClientConfigs, Metadata, NetworkClient, NodeApiVersions}
+import org.apache.kafka.clients.consumer.internals.{ConsumerNetworkClient, RequestFuture}
+import org.apache.kafka.common.config.ConfigDef.ValidString._
+import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
+import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}
+import org.apache.kafka.common.errors.AuthenticationException
+import org.apache.kafka.common.internals.ClusterResourceListeners
+import org.apache.kafka.common.metrics.Metrics
+import org.apache.kafka.common.network.Selector
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.utils.LogContext
+import org.apache.kafka.common.utils.{KafkaThread, Time}
+import org.apache.kafka.common.Node
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection
+import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ApiVersionsRequest, ApiVersionsResponse, MetadataRequest, MetadataResponse}
+
+import scala.collection.JavaConverters._
+import scala.util.{Failure, Success, Try}
+
+/**
+ * A command for retrieving broker version information.
+ */
+object BrokerApiVersionsCommand {
+
+  def main(args: Array[String]): Unit = {
+    execute(args, System.out)
+  }
+
+  def execute(args: Array[String], out: PrintStream): Unit = {
+    val opts = new BrokerVersionCommandOptions(args)
+    val adminClient = createAdminClient(opts)
+    adminClient.awaitBrokers()
+    val brokerMap = adminClient.listAllBrokerVersionInfo()
+    brokerMap.foreach { case (broker, versionInfoOrError) =>
+      versionInfoOrError match {
+        case Success(v) => out.print(s"${broker} -> ${v.toString(true)}\n")
+        case Failure(v) => out.print(s"${broker} -> ERROR: ${v}\n")
+      }
+    }
+    adminClient.close()
+  }
+
+  private def createAdminClient(opts: BrokerVersionCommandOptions): AdminClient = {
+    val props = if (opts.options.has(opts.commandConfigOpt))
+      Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+    else
+      new Properties()
+    props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+    AdminClient.create(props)
+  }
+
+  class BrokerVersionCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    val BootstrapServerDoc = "REQUIRED: The server to connect to."
+    val CommandConfigDoc = "A property file containing configs to be passed to Admin Client."
+
+    val commandConfigOpt = parser.accepts("command-config", CommandConfigDoc)
+                                 .withRequiredArg
+                                 .describedAs("command config property file")
+                                 .ofType(classOf[String])
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", BootstrapServerDoc)
+                                   .withRequiredArg
+                                   .describedAs("server(s) to use for bootstrapping")
+                                   .ofType(classOf[String])
+    options = parser.parse(args : _*)
+    checkArgs()
+
+    def checkArgs(): Unit = {
+      CommandLineUtils.printHelpAndExitIfNeeded(this, "This tool helps to retrieve broker version information.")
+      // check required args
+      CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt)
+    }
+  }
+
+  // org.apache.kafka.clients.admin.AdminClient doesn't currently expose a way to retrieve the supported api versions.
+  // We inline the bits we need from kafka.admin.AdminClient so that we can delete it.
+  private class AdminClient(val time: Time,
+                            val requestTimeoutMs: Int,
+                            val retryBackoffMs: Long,
+                            val client: ConsumerNetworkClient,
+                            val bootstrapBrokers: List[Node]) extends Logging {
+
+    @volatile var running: Boolean = true
+    val pendingFutures = new ConcurrentLinkedQueue[RequestFuture[ClientResponse]]()
+
+    val networkThread = new KafkaThread("admin-client-network-thread", () => {
+      try {
+        while (running)
+          client.poll(time.timer(Long.MaxValue))
+      } catch {
+        case t: Throwable =>
+          error("admin-client-network-thread exited", t)
+      } finally {
+        pendingFutures.asScala.foreach { future =>
+          try {
+            future.raise(Errors.UNKNOWN_SERVER_ERROR)
+          } catch {
+            case _: IllegalStateException => // It is OK if the future has been completed
+          }
+        }
+        pendingFutures.clear()
+      }
+    }, true)
+
+    networkThread.start()
+
+    private def send(target: Node,
+                     api: ApiKeys,
+                     request: AbstractRequest.Builder[_ <: AbstractRequest]): AbstractResponse = {
+      val future: RequestFuture[ClientResponse] = client.send(target, request)
+      pendingFutures.add(future)
+      future.awaitDone(Long.MaxValue, TimeUnit.MILLISECONDS)
+      pendingFutures.remove(future)
+      if (future.succeeded())
+        future.value().responseBody()
+      else
+        throw future.exception()
+    }
+
+    private def sendAnyNode(api: ApiKeys, request: AbstractRequest.Builder[_ <: AbstractRequest]): AbstractResponse = {
+      bootstrapBrokers.foreach { broker =>
+        try {
+          return send(broker, api, request)
+        } catch {
+          case e: AuthenticationException =>
+            throw e
+          case e: Exception =>
+            debug(s"Request $api failed against node $broker", e)
+        }
+      }
+      throw new RuntimeException(s"Request $api failed on brokers $bootstrapBrokers")
+    }
+
+    private def getApiVersions(node: Node): ApiVersionsResponseKeyCollection = {
+      val response = send(node, ApiKeys.API_VERSIONS, new ApiVersionsRequest.Builder()).asInstanceOf[ApiVersionsResponse]
+      Errors.forCode(response.data.errorCode).maybeThrow()
+      response.data.apiKeys
+    }
+
+    /**
+     * Wait until there is a non-empty list of brokers in the cluster.
+     */
+    def awaitBrokers(): Unit = {
+      var nodes = List[Node]()
+      do {
+        nodes = findAllBrokers()
+        if (nodes.isEmpty)
+          Thread.sleep(50)
+      } while (nodes.isEmpty)
+    }
+
+    private def findAllBrokers(): List[Node] = {
+      val request = MetadataRequest.Builder.allTopics()
+      val response = sendAnyNode(ApiKeys.METADATA, request).asInstanceOf[MetadataResponse]
+      val errors = response.errors
+      if (!errors.isEmpty)
+        debug(s"Metadata request contained errors: $errors")
+      response.cluster.nodes.asScala.toList
+    }
+
+    def listAllBrokerVersionInfo(): Map[Node, Try[NodeApiVersions]] =
+      findAllBrokers().map { broker =>
+        broker -> Try[NodeApiVersions](new NodeApiVersions(getApiVersions(broker)))
+      }.toMap
+
+    def close(): Unit = {
+      running = false
+      try {
+        client.close()
+      } catch {
+        case e: IOException =>
+          error("Exception closing nioSelector:", e)
+      }
+    }
+
+  }
+
+  private object AdminClient {
+    val DefaultConnectionMaxIdleMs = 9 * 60 * 1000
+    val DefaultRequestTimeoutMs = 5000
+    val DefaultMaxInFlightRequestsPerConnection = 100
+    val DefaultReconnectBackoffMs = 50
+    val DefaultReconnectBackoffMax = 50
+    val DefaultSendBufferBytes = 128 * 1024
+    val DefaultReceiveBufferBytes = 32 * 1024
+    val DefaultRetryBackoffMs = 100
+
+    val AdminClientIdSequence = new AtomicInteger(1)
+    val AdminConfigDef = {
+      val config = new ConfigDef()
+        .define(
+          CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
+          Type.LIST,
+          Importance.HIGH,
+          CommonClientConfigs.BOOTSTRAP_SERVERS_DOC)
+        .define(CommonClientConfigs.CLIENT_DNS_LOOKUP_CONFIG,
+          Type.STRING,
+          ClientDnsLookup.DEFAULT.toString,
+          in(ClientDnsLookup.DEFAULT.toString,
+            ClientDnsLookup.USE_ALL_DNS_IPS.toString,
+            ClientDnsLookup.RESOLVE_CANONICAL_BOOTSTRAP_SERVERS_ONLY.toString),
+          Importance.MEDIUM,
+          CommonClientConfigs.CLIENT_DNS_LOOKUP_DOC)
+        .define(
+          CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
+          ConfigDef.Type.STRING,
+          CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL,
+          ConfigDef.Importance.MEDIUM,
+          CommonClientConfigs.SECURITY_PROTOCOL_DOC)
+        .define(
+          CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG,
+          ConfigDef.Type.INT,
+          DefaultRequestTimeoutMs,
+          ConfigDef.Importance.MEDIUM,
+          CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC)
+        .define(
+          CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG,
+          ConfigDef.Type.LONG,
+          DefaultRetryBackoffMs,
+          ConfigDef.Importance.MEDIUM,
+          CommonClientConfigs.RETRY_BACKOFF_MS_DOC)
+        .withClientSslSupport()
+        .withClientSaslSupport()
+      config
+    }
+
+    class AdminConfig(originals: Map[_,_]) extends AbstractConfig(AdminConfigDef, originals.asJava, false)
+
+    def createSimplePlaintext(brokerUrl: String): AdminClient = {
+      val config = Map(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG -> brokerUrl)
+      create(new AdminConfig(config))
+    }
+
+    def create(props: Properties): AdminClient = create(props.asScala.toMap)
+
+    def create(props: Map[String, _]): AdminClient = create(new AdminConfig(props))
+
+    def create(config: AdminConfig): AdminClient = {
+      val clientId = "admin-" + AdminClientIdSequence.getAndIncrement()
+      val logContext = new LogContext(s"[LegacyAdminClient clientId=$clientId] ")
+      val time = Time.SYSTEM
+      val metrics = new Metrics(time)
+      val metadata = new Metadata(100L, 60 * 60 * 1000L, logContext,
+        new ClusterResourceListeners)
+      val channelBuilder = ClientUtils.createChannelBuilder(config, time, logContext)
+      val requestTimeoutMs = config.getInt(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG)
+      val retryBackoffMs = config.getLong(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG)
+
+      val brokerUrls = config.getList(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)
+      val clientDnsLookup = config.getString(CommonClientConfigs.CLIENT_DNS_LOOKUP_CONFIG)
+      val brokerAddresses = ClientUtils.parseAndValidateAddresses(brokerUrls, clientDnsLookup)
+      metadata.bootstrap(brokerAddresses)
+
+      val selector = new Selector(
+        DefaultConnectionMaxIdleMs,
+        metrics,
+        time,
+        "admin",
+        channelBuilder,
+        logContext)
+
+      val networkClient = new NetworkClient(
+        selector,
+        metadata,
+        clientId,
+        DefaultMaxInFlightRequestsPerConnection,
+        DefaultReconnectBackoffMs,
+        DefaultReconnectBackoffMax,
+        DefaultSendBufferBytes,
+        DefaultReceiveBufferBytes,
+        requestTimeoutMs,
+        ClientDnsLookup.DEFAULT,
+        time,
+        true,
+        new ApiVersions,
+        logContext)
+
+      val highLevelClient = new ConsumerNetworkClient(
+        logContext,
+        networkClient,
+        metadata,
+        time,
+        retryBackoffMs,
+        requestTimeoutMs,
+        Integer.MAX_VALUE)
+
+      new AdminClient(
+        time,
+        requestTimeoutMs,
+        retryBackoffMs,
+        highLevelClient,
+        metadata.fetch.nodes.asScala.toList)
+    }
+  }
+
+}

core/src/main/scala/kafka/admin/BrokerMetadata.scala

@@ -0,0 +1,23 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+  * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+  * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+  * License. You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+  * specific language governing permissions and limitations under the License.
+  */
+
+package kafka.admin
+
+/**
+  * Broker metadata used by admin tools.
+  *
+  * @param id an integer that uniquely identifies this broker
+  * @param rack the rack of the broker, which is used to in rack aware partition assignment for fault tolerance.
+  *             Examples: "RACK1", "us-east-1d"
+  */
+case class BrokerMetadata(id: Int, rack: Option[String])

core/src/main/scala/kafka/admin/ConfigCommand.scala

@@ -0,0 +1,737 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+
+import com.didichuxing.datachannel.kafka.config.{HAClusterConfig, HAGroupConfig, HATopicConfig}
+
+import java.util.concurrent.TimeUnit
+import java.util.{Collections, Properties}
+import joptsimple._
+import kafka.common.Config
+import kafka.log.LogConfig
+import kafka.server.{ConfigEntityName, ConfigType, Defaults, DynamicBrokerConfig, DynamicConfig, KafkaConfig}
+import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Exit, PasswordEncoder}
+import kafka.utils.Implicits._
+import kafka.zk.{AdminZkClient, KafkaZkClient}
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.clients.admin.{Admin, AlterConfigOp, AlterConfigsOptions, ConfigEntry, DescribeClusterOptions, DescribeConfigsOptions, ListTopicsOptions, Config => JConfig}
+import org.apache.kafka.common.config.{ConfigResource}
+import org.apache.kafka.common.config.types.Password
+import org.apache.kafka.common.errors.InvalidConfigurationException
+import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.security.scram.internals.{ScramCredentialUtils, ScramFormatter, ScramMechanism}
+import org.apache.kafka.common.utils.{Sanitizer, Time, Utils}
+import org.apache.zookeeper.client.ZKClientConfig
+
+import scala.collection.JavaConverters._
+import scala.collection._
+
+
+/**
+ * This script can be used to change configs for topics/clients/users/brokers dynamically
+ * An entity described or altered by the command may be one of:
+ * <ul>
+ *     <li> topic: --topic <topic> OR --entity-type topics --entity-name <topic>
+ *     <li> client: --client <client> OR --entity-type clients --entity-name <client-id>
+ *     <li> user: --user <user-principal> OR --entity-type users --entity-name <user-principal>
+ *     <li> <user, client>: --user <user-principal> --client <client-id> OR
+ *                          --entity-type users --entity-name <user-principal> --entity-type clients --entity-name <client-id>
+ *     <li> broker: --broker <broker-id> OR --entity-type brokers --entity-name <broker-id>
+ *     <li> broker-logger: --broker-logger <broker-id> OR --entity-type broker-loggers --entity-name <broker-id>
+ * </ul>
+ * --user-defaults, --client-defaults, or --broker-defaults may be when describing or altering default configuration for users,
+ * clients, and brokers, respectively. Alternatively, --entity-default may be used instead of --entity-name.
+ *
+ */
+object ConfigCommand extends Config {
+
+  val BrokerDefaultEntityName = ""
+  val BrokerLoggerConfigType = "broker-loggers"
+  val BrokerSupportedConfigTypes = Seq(ConfigType.Topic, ConfigType.Broker, BrokerLoggerConfigType)
+  val DefaultScramIterations = 4096
+  // Dynamic broker configs can only be updated using the new AdminClient once brokers have started
+  // so that configs may be fully validated. Prior to starting brokers, updates may be performed using
+  // ZooKeeper for bootstrapping. This allows all password configs to be stored encrypted in ZK,
+  // avoiding clear passwords in server.properties. For consistency with older versions, quota-related
+  // broker configs can still be updated using ZooKeeper at any time. ConfigCommand will be migrated
+  // to the new AdminClient later for these configs (KIP-248).
+  val BrokerConfigsUpdatableUsingZooKeeperWhileBrokerRunning = Set(
+    DynamicConfig.Broker.LeaderReplicationThrottledRateProp,
+    DynamicConfig.Broker.FollowerReplicationThrottledRateProp,
+    DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp)
+
+  def main(args: Array[String]): Unit = {
+    try {
+      val opts = new ConfigCommandOptions(args)
+
+      CommandLineUtils.printHelpAndExitIfNeeded(opts, "This tool helps to manipulate and describe entity config for a topic, client, user, cluster or broker")
+
+      opts.checkArgs()
+
+      if (opts.options.has(opts.zkConnectOpt)) {
+        println(s"Warning: --zookeeper is deprecated and will be removed in a future version of Kafka.")
+        println(s"Use --bootstrap-server instead to specify a broker to connect to.")
+        processCommandWithZk(opts.options.valueOf(opts.zkConnectOpt), opts)
+      } else {
+        processCommand(opts)
+      }
+    } catch {
+      case e @ (_: IllegalArgumentException | _: InvalidConfigurationException | _: OptionException) =>
+        logger.debug(s"Failed config command with args '${args.mkString(" ")}'", e)
+        System.err.println(e.getMessage)
+        Exit.exit(1)
+
+      case t: Throwable =>
+        logger.debug(s"Error while executing config command with args '${args.mkString(" ")}'", t)
+        System.err.println(s"Error while executing config command with args '${args.mkString(" ")}'")
+        t.printStackTrace(System.err)
+        Exit.exit(1)
+    }
+  }
+
+  private def processCommandWithZk(zkConnectString: String, opts: ConfigCommandOptions): Unit = {
+    val zkClientConfig = ZkSecurityMigrator.createZkClientConfigFromOption(opts.options, opts.zkTlsConfigFile)
+      .getOrElse(new ZKClientConfig())
+    val zkClient = KafkaZkClient(zkConnectString, JaasUtils.isZkSaslEnabled || KafkaConfig.zkTlsClientAuthEnabled(zkClientConfig), 30000, 30000,
+      Int.MaxValue, Time.SYSTEM, zkClientConfig = Some(zkClientConfig))
+    val adminZkClient = new AdminZkClient(zkClient)
+    try {
+      if (opts.options.has(opts.alterOpt))
+        alterConfigWithZk(zkClient, opts, adminZkClient)
+      else if (opts.options.has(opts.describeOpt))
+        describeConfigWithZk(zkClient, opts, adminZkClient)
+    } finally {
+      zkClient.close()
+    }
+  }
+
+  private[admin] def alterConfigWithZk(zkClient: KafkaZkClient, opts: ConfigCommandOptions, adminZkClient: AdminZkClient): Unit = {
+    val configsToBeAdded = parseConfigsToBeAdded(opts)
+    val configsToBeDeleted = parseConfigsToBeDeleted(opts)
+    val entity = parseEntity(opts)
+    val entityType = entity.root.entityType
+    val entityName = entity.fullSanitizedName
+
+    if (entityType == ConfigType.User)
+      preProcessScramCredentials(configsToBeAdded)
+    else if (entityType == ConfigType.Broker) {
+      // Replication quota configs may be updated using ZK at any time. Other dynamic broker configs
+      // may be updated using ZooKeeper only if the corresponding broker is not running. Dynamic broker
+      // configs at cluster-default level may be configured using ZK only if there are no brokers running.
+      val dynamicBrokerConfigs = configsToBeAdded.asScala.keySet.filterNot(BrokerConfigsUpdatableUsingZooKeeperWhileBrokerRunning.contains)
+      if (dynamicBrokerConfigs.nonEmpty) {
+        val perBrokerConfig = entityName != ConfigEntityName.Default
+        val errorMessage = s"--bootstrap-server option must be specified to update broker configs $dynamicBrokerConfigs."
+        val info = "Broker configuration updates using ZooKeeper are supported for bootstrapping before brokers" +
+          " are started to enable encrypted password configs to be stored in ZooKeeper."
+        if (perBrokerConfig) {
+          adminZkClient.parseBroker(entityName).foreach { brokerId =>
+            require(zkClient.getBroker(brokerId).isEmpty, s"$errorMessage when broker $entityName is running. $info")
+          }
+        } else {
+          require(zkClient.getAllBrokersInCluster.isEmpty, s"$errorMessage for default cluster if any broker is running. $info")
+        }
+        preProcessBrokerConfigs(configsToBeAdded, perBrokerConfig)
+      }
+    } else if (entityType == ConfigType.HATopic) {
+      if (configsToBeAdded.containsKey(HATopicConfig.DIDI_HA_REMOTE_CLUSTER)) {
+        val haClusterId = configsToBeAdded.getProperty(HATopicConfig.DIDI_HA_REMOTE_CLUSTER)
+        val haConfig = adminZkClient.fetchEntityConfig(ConfigType.HACluster, haClusterId)
+        require(!haConfig.isEmpty, s"does not exist ha-clusters $haClusterId.")
+        val haTopicConfig = adminZkClient.fetchEntityConfig(ConfigType.HATopic, entityName)
+        if (!haTopicConfig.isEmpty) {
+          require(haTopicConfig.getProperty(HATopicConfig.DIDI_HA_REMOTE_CLUSTER) == null, s"${HATopicConfig.DIDI_HA_REMOTE_CLUSTER} does not support updating. only add after deleting.")
+        }
+      } else if (configsToBeAdded.containsKey(HATopicConfig.DIDI_HA_REMOTE_TOPIC)) {
+        val remoteTopic = configsToBeAdded.getProperty(HATopicConfig.DIDI_HA_REMOTE_TOPIC)
+        val configs = adminZkClient.fetchAllEntityConfigs(ConfigType.HATopic).filterNot(_._1.contains(entityName))
+          .filter(_._2.getProperty(HATopicConfig.DIDI_HA_REMOTE_TOPIC, "").equals(remoteTopic))
+        require(configs.isEmpty, s"remoteTopic $remoteTopic reference already exists.")
+      }
+    }
+
+    // compile the final set of configs
+    val configs = adminZkClient.fetchEntityConfig(entityType, entityName)
+
+    // fail the command if any of the configs to be deleted does not exist
+    val invalidConfigs = configsToBeDeleted.filterNot(configs.containsKey(_))
+    if (invalidConfigs.nonEmpty)
+      throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}")
+
+
+
+    configs ++= configsToBeAdded
+    configsToBeDeleted.foreach(configs.remove(_))
+
+    //存在Topic或User对某集群的依赖时不能删除该集群配置
+    if (entityType == ConfigType.HACluster && configsToBeDeleted.contains(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG))
+      adminZkClient.validClusterConfigsDeletable(entityName)
+
+    adminZkClient.changeConfigs(entityType, entityName, configs)
+
+    println(s"Completed updating config for entity: $entity.")
+  }
+
+  private def preProcessScramCredentials(configsToBeAdded: Properties): Unit = {
+    def scramCredential(mechanism: ScramMechanism, credentialStr: String): String = {
+      val pattern = "(?:iterations=([0-9]*),)?password=(.*)".r
+      val (iterations, password) = credentialStr match {
+          case pattern(iterations, password) => (if (iterations != null) iterations.toInt else DefaultScramIterations, password)
+          case _ => throw new IllegalArgumentException(s"Invalid credential property $mechanism=$credentialStr")
+        }
+      if (iterations < mechanism.minIterations())
+        throw new IllegalArgumentException(s"Iterations $iterations is less than the minimum ${mechanism.minIterations()} required for $mechanism")
+      val credential = new ScramFormatter(mechanism).generateCredential(password, iterations)
+      ScramCredentialUtils.credentialToString(credential)
+    }
+    for (mechanism <- ScramMechanism.values) {
+      configsToBeAdded.getProperty(mechanism.mechanismName) match {
+        case null =>
+        case value =>
+          configsToBeAdded.setProperty(mechanism.mechanismName, scramCredential(mechanism, value))
+      }
+    }
+  }
+
+  private[admin] def createPasswordEncoder(encoderConfigs: Map[String, String]): PasswordEncoder = {
+    encoderConfigs.get(KafkaConfig.PasswordEncoderSecretProp)
+    val encoderSecret = encoderConfigs.getOrElse(KafkaConfig.PasswordEncoderSecretProp,
+      throw new IllegalArgumentException("Password encoder secret not specified"))
+    new PasswordEncoder(new Password(encoderSecret),
+      None,
+      encoderConfigs.get(KafkaConfig.PasswordEncoderCipherAlgorithmProp).getOrElse(Defaults.PasswordEncoderCipherAlgorithm),
+      encoderConfigs.get(KafkaConfig.PasswordEncoderKeyLengthProp).map(_.toInt).getOrElse(Defaults.PasswordEncoderKeyLength),
+      encoderConfigs.get(KafkaConfig.PasswordEncoderIterationsProp).map(_.toInt).getOrElse(Defaults.PasswordEncoderIterations))
+  }
+
+  /**
+   * Pre-process broker configs provided to convert them to persistent format.
+   * Password configs are encrypted using the secret `KafkaConfig.PasswordEncoderSecretProp`.
+   * The secret is removed from `configsToBeAdded` and will not be persisted in ZooKeeper.
+   */
+  private def preProcessBrokerConfigs(configsToBeAdded: Properties, perBrokerConfig: Boolean): Unit = {
+    val passwordEncoderConfigs = new Properties
+    passwordEncoderConfigs ++= configsToBeAdded.asScala.filter { case (key, _) => key.startsWith("password.encoder.") }
+    if (!passwordEncoderConfigs.isEmpty) {
+      info(s"Password encoder configs ${passwordEncoderConfigs.keySet} will be used for encrypting" +
+        " passwords, but will not be stored in ZooKeeper.")
+      passwordEncoderConfigs.asScala.keySet.foreach(configsToBeAdded.remove)
+    }
+
+    DynamicBrokerConfig.validateConfigs(configsToBeAdded, perBrokerConfig)
+    val passwordConfigs = configsToBeAdded.asScala.keySet.filter(DynamicBrokerConfig.isPasswordConfig)
+    if (passwordConfigs.nonEmpty) {
+      require(passwordEncoderConfigs.containsKey(KafkaConfig.PasswordEncoderSecretProp),
+        s"${KafkaConfig.PasswordEncoderSecretProp} must be specified to update $passwordConfigs." +
+          " Other password encoder configs like cipher algorithm and iterations may also be specified" +
+          " to override the default encoding parameters. Password encoder configs will not be persisted" +
+          " in ZooKeeper."
+      )
+
+      val passwordEncoder = createPasswordEncoder(passwordEncoderConfigs.asScala)
+      passwordConfigs.foreach { configName =>
+        val encodedValue = passwordEncoder.encode(new Password(configsToBeAdded.getProperty(configName)))
+        configsToBeAdded.setProperty(configName, encodedValue)
+      }
+    }
+  }
+
+  private def describeConfigWithZk(zkClient: KafkaZkClient, opts: ConfigCommandOptions, adminZkClient: AdminZkClient): Unit = {
+    val configEntity = parseEntity(opts)
+    val describeAllUsers = configEntity.root.entityType == ConfigType.User && !configEntity.root.sanitizedName.isDefined && !configEntity.child.isDefined
+    val entities = configEntity.getAllEntities(zkClient)
+    for (entity <- entities) {
+      val configs = adminZkClient.fetchEntityConfig(entity.root.entityType, entity.fullSanitizedName)
+      // When describing all users, don't include empty user nodes with only <user, client> quota overrides.
+      if (!configs.isEmpty || !describeAllUsers) {
+        println("Configs for %s are %s"
+          .format(entity, configs.asScala.map(kv => kv._1 + "=" + kv._2).mkString(",")))
+      }
+    }
+  }
+
+  private[admin] def parseConfigsToBeAdded(opts: ConfigCommandOptions): Properties = {
+    val props = new Properties
+    if (opts.options.has(opts.addConfig)) {
+      // Split list by commas, but avoid those in [], then into KV pairs
+      // Each KV pair is of format key=value, split them into key and value, using -1 as the limit for split() to
+      // include trailing empty strings. This is to support empty value (e.g. 'ssl.endpoint.identification.algorithm=')
+      val pattern = "(?=[^\\]]*(?:\\[|$))"
+      val configsToBeAdded = opts.options.valueOf(opts.addConfig)
+        .split("," + pattern)
+        .map(_.split("""\s*=\s*""" + pattern, -1))
+      require(configsToBeAdded.forall(config => config.length == 2), "Invalid entity config: all configs to be added must be in the format \"key=val\".")
+      //Create properties, parsing square brackets from values if necessary
+      configsToBeAdded.foreach(pair => props.setProperty(pair(0).trim, pair(1).replaceAll("\\[?\\]?", "").trim))
+      if (props.containsKey(LogConfig.MessageFormatVersionProp)) {
+        println(s"WARNING: The configuration ${LogConfig.MessageFormatVersionProp}=${props.getProperty(LogConfig.MessageFormatVersionProp)} is specified. " +
+          s"This configuration will be ignored if the version is newer than the inter.broker.protocol.version specified in the broker.")
+      }
+    }
+    props
+  }
+
+  private[admin] def parseConfigsToBeDeleted(opts: ConfigCommandOptions): Seq[String] = {
+    if (opts.options.has(opts.deleteConfig)) {
+      val configsToBeDeleted = opts.options.valuesOf(opts.deleteConfig).asScala.map(_.trim())
+      val propsToBeDeleted = new Properties
+      configsToBeDeleted.foreach(propsToBeDeleted.setProperty(_, ""))
+      configsToBeDeleted
+    }
+    else
+      Seq.empty
+  }
+
+  private def processCommand(opts: ConfigCommandOptions): Unit = {
+    val props = if (opts.options.has(opts.commandConfigOpt))
+      Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+    else
+      new Properties()
+    props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+    val adminClient = Admin.create(props)
+
+    if (opts.entityTypes.size != 1)
+      throw new IllegalArgumentException(s"Exactly one entity type (out of ${BrokerSupportedConfigTypes.mkString(",")}) must be specified with --bootstrap-server")
+
+    val entityNames = opts.entityNames
+    if (entityNames.size > 1)
+      throw new IllegalArgumentException(s"At most one entity name must be specified with --bootstrap-server")
+    else if (opts.options.has(opts.alterOpt) && entityNames.size != 1)
+      throw new IllegalArgumentException(s"Exactly one entity name must be specified with --bootstrap-server for --alter")
+
+    try {
+      if (opts.options.has(opts.alterOpt))
+        alterConfig(adminClient, opts)
+      else if (opts.options.has(opts.describeOpt))
+        describeConfig(adminClient, opts)
+    } finally {
+      adminClient.close()
+    }
+  }
+
+  private[admin] def alterConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = {
+    val entityType = opts.entityTypes.head
+    val entityName = opts.entityNames.head
+    val configsToBeAdded = parseConfigsToBeAdded(opts).asScala.map { case (k, v) => (k, new ConfigEntry(k, v)) }
+    val configsToBeDeleted = parseConfigsToBeDeleted(opts)
+
+    entityType match {
+      case ConfigType.Topic =>
+        val oldConfig = getConfig(adminClient, entityType, entityName, includeSynonyms = false, describeAll = false)
+          .map { entry => (entry.name, entry) }.toMap
+
+        // fail the command if any of the configs to be deleted does not exist
+        val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains)
+        if (invalidConfigs.nonEmpty)
+          throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}")
+
+        val configResource = new ConfigResource(ConfigResource.Type.TOPIC, entityName)
+        val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false)
+        val alterEntries = (configsToBeAdded.values.map(new AlterConfigOp(_, AlterConfigOp.OpType.SET))
+          ++ configsToBeDeleted.map { k => new AlterConfigOp(new ConfigEntry(k, ""), AlterConfigOp.OpType.DELETE) }
+        ).asJavaCollection
+        adminClient.incrementalAlterConfigs(Map(configResource -> alterEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS)
+
+      case ConfigType.Broker =>
+        val oldConfig = getConfig(adminClient, entityType, entityName, includeSynonyms = false, describeAll = false)
+          .map { entry => (entry.name, entry) }.toMap
+
+        // fail the command if any of the configs to be deleted does not exist
+        val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains)
+        if (invalidConfigs.nonEmpty)
+          throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}")
+
+        val newEntries = oldConfig ++ configsToBeAdded -- configsToBeDeleted
+        val sensitiveEntries = newEntries.filter(_._2.value == null)
+        if (sensitiveEntries.nonEmpty)
+          throw new InvalidConfigurationException(s"All sensitive broker config entries must be specified for --alter, missing entries: ${sensitiveEntries.keySet}")
+        val newConfig = new JConfig(newEntries.asJava.values)
+
+        val configResource = new ConfigResource(ConfigResource.Type.BROKER, entityName)
+        val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false)
+        adminClient.alterConfigs(Map(configResource -> newConfig).asJava, alterOptions).all().get(60, TimeUnit.SECONDS)
+
+      case BrokerLoggerConfigType =>
+        val validLoggers = getConfig(adminClient, entityType, entityName, includeSynonyms = true, describeAll = false).map(_.name)
+        // fail the command if any of the configured broker loggers do not exist
+        val invalidBrokerLoggers = configsToBeDeleted.filterNot(validLoggers.contains) ++ configsToBeAdded.keys.filterNot(validLoggers.contains)
+        if (invalidBrokerLoggers.nonEmpty)
+          throw new InvalidConfigurationException(s"Invalid broker logger(s): ${invalidBrokerLoggers.mkString(",")}")
+
+        val configResource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, entityName)
+        val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false)
+        val alterLogLevelEntries = (configsToBeAdded.values.map(new AlterConfigOp(_, AlterConfigOp.OpType.SET))
+          ++ configsToBeDeleted.map { k => new AlterConfigOp(new ConfigEntry(k, ""), AlterConfigOp.OpType.DELETE) }
+        ).asJavaCollection
+        adminClient.incrementalAlterConfigs(Map(configResource -> alterLogLevelEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS)
+
+      case _ => throw new IllegalArgumentException(s"Unsupported entity type: $entityType")
+    }
+
+    if (entityName.nonEmpty)
+      println(s"Completed updating config for ${entityType.dropRight(1)} $entityName.")
+    else
+      println(s"Completed updating default config for $entityType in the cluster.")
+  }
+
+  private[admin] def describeConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = {
+    val entityType = opts.entityTypes.head
+    val entityName = opts.entityNames.headOption
+    val describeAll = opts.options.has(opts.allOpt)
+
+    val entities = entityName
+      .map(name => List(name))
+      .getOrElse(entityType match {
+        case ConfigType.Topic =>
+          adminClient.listTopics(new ListTopicsOptions().listInternal(true)).names().get().asScala.toSeq
+        case ConfigType.Broker | BrokerLoggerConfigType =>
+          adminClient.describeCluster(new DescribeClusterOptions()).nodes().get().asScala.map(_.idString).toSeq :+ BrokerDefaultEntityName
+      })
+
+    entities.foreach { entity =>
+      entity match {
+        case BrokerDefaultEntityName =>
+          println(s"Default configs for $entityType in the cluster are:")
+        case _ =>
+          val configSourceStr = if (describeAll) "All" else "Dynamic"
+          println(s"$configSourceStr configs for ${entityType.dropRight(1)} $entity are:")
+      }
+      getConfig(adminClient, entityType, entity, includeSynonyms = true, describeAll).foreach { entry =>
+        val synonyms = entry.synonyms.asScala.map(synonym => s"${synonym.source}:${synonym.name}=${synonym.value}").mkString(", ")
+        println(s"  ${entry.name}=${entry.value} sensitive=${entry.isSensitive} synonyms={$synonyms}")
+      }
+    }
+  }
+
+  private def getConfig(adminClient: Admin, entityType: String, entityName: String, includeSynonyms: Boolean, describeAll: Boolean) = {
+    def validateBrokerId(): Unit = try entityName.toInt catch {
+      case _: NumberFormatException =>
+        throw new IllegalArgumentException(s"The entity name for $entityType must be a valid integer broker id, found: $entityName")
+    }
+
+    val (configResourceType, dynamicConfigSource) = entityType match {
+      case ConfigType.Topic =>
+        if (!entityName.isEmpty)
+          Topic.validate(entityName)
+        (ConfigResource.Type.TOPIC, Some(ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG))
+      case ConfigType.Broker => entityName match {
+        case BrokerDefaultEntityName =>
+          (ConfigResource.Type.BROKER, Some(ConfigEntry.ConfigSource.DYNAMIC_DEFAULT_BROKER_CONFIG))
+        case _ =>
+          validateBrokerId()
+          (ConfigResource.Type.BROKER, Some(ConfigEntry.ConfigSource.DYNAMIC_BROKER_CONFIG))
+      }
+      case BrokerLoggerConfigType =>
+        if (!entityName.isEmpty)
+          validateBrokerId()
+        (ConfigResource.Type.BROKER_LOGGER, None)
+    }
+
+    val configSourceFilter = if (describeAll)
+      None
+    else
+      dynamicConfigSource
+
+    val configResource = new ConfigResource(configResourceType, entityName)
+    val describeOptions = new DescribeConfigsOptions().includeSynonyms(includeSynonyms)
+    val configs = adminClient.describeConfigs(Collections.singleton(configResource), describeOptions)
+      .all.get(30, TimeUnit.SECONDS)
+    configs.get(configResource).entries.asScala
+      .filter(entry => configSourceFilter match {
+        case Some(configSource) => entry.source == configSource
+        case None => true
+      }).toSeq
+  }
+
+  case class Entity(entityType: String, sanitizedName: Option[String]) {
+    val entityPath = sanitizedName match {
+      case Some(n) => entityType + "/" + n
+      case None => entityType
+    }
+    override def toString: String = {
+      val typeName = entityType match {
+        case ConfigType.User => "user-principal"
+        case ConfigType.Client => "client-id"
+        case ConfigType.Topic => "topic"
+        case t => t
+      }
+      sanitizedName match {
+        case Some(ConfigEntityName.Default) => "default " + typeName
+        case Some(n) =>
+          val desanitized = if (entityType == ConfigType.User || entityType == ConfigType.Client) Sanitizer.desanitize(n) else n
+          s"$typeName '$desanitized'"
+        case None => entityType
+      }
+    }
+  }
+
+  case class ConfigEntity(root: Entity, child: Option[Entity]) {
+    val fullSanitizedName = root.sanitizedName.getOrElse("") + child.map(s => "/" + s.entityPath).getOrElse("")
+
+    def getAllEntities(zkClient: KafkaZkClient) : Seq[ConfigEntity] = {
+      // Describe option examples:
+      //   Describe entity with specified name:
+      //     --entity-type topics --entity-name topic1 (topic1)
+      //   Describe all entities of a type (topics/brokers/users/clients):
+      //     --entity-type topics (all topics)
+      //   Describe <user, client> quotas:
+      //     --entity-type users --entity-name user1 --entity-type clients --entity-name client2 (<user1, client2>)
+      //     --entity-type users --entity-name userA --entity-type clients (all clients of userA)
+      //     --entity-type users --entity-type clients (all <user, client>s))
+      //   Describe default quotas:
+      //     --entity-type users --entity-default (Default user)
+      //     --entity-type users --entity-default --entity-type clients --entity-default (Default <user, client>)
+      (root.sanitizedName, child) match {
+        case (None, _) =>
+          val rootEntities = zkClient.getAllEntitiesWithConfig(root.entityType)
+                                   .map(name => ConfigEntity(Entity(root.entityType, Some(name)), child))
+          child match {
+            case Some(s) =>
+                rootEntities.flatMap(rootEntity =>
+                  ConfigEntity(rootEntity.root, Some(Entity(s.entityType, None))).getAllEntities(zkClient))
+            case None => rootEntities
+          }
+        case (_, Some(childEntity)) =>
+          childEntity.sanitizedName match {
+            case Some(_) => Seq(this)
+            case None =>
+                zkClient.getAllEntitiesWithConfig(root.entityPath + "/" + childEntity.entityType)
+                       .map(name => ConfigEntity(root, Some(Entity(childEntity.entityType, Some(name)))))
+
+          }
+        case (_, None) =>
+          Seq(this)
+      }
+    }
+
+    override def toString: String = {
+      root.toString + child.map(s => ", " + s.toString).getOrElse("")
+    }
+  }
+
+  private[admin] def parseEntity(opts: ConfigCommandOptions): ConfigEntity = {
+    val entityTypes = opts.entityTypes
+    val entityNames = opts.entityNames
+    if (entityTypes.head == ConfigType.User || entityTypes.head == ConfigType.Client)
+      parseQuotaEntity(opts, entityTypes, entityNames)
+    else {
+      // Exactly one entity type and at-most one entity name expected for other entities
+      val name = entityNames.headOption match {
+        case Some("") => Some(ConfigEntityName.Default)
+        case v => v
+      }
+      ConfigEntity(Entity(entityTypes.head, name), None)
+    }
+  }
+
+  private def parseQuotaEntity(opts: ConfigCommandOptions, types: List[String], names: List[String]): ConfigEntity = {
+    if (opts.options.has(opts.alterOpt) && names.size != types.size)
+      throw new IllegalArgumentException("--entity-name or --entity-default must be specified with each --entity-type for --alter")
+
+    val reverse = types.size == 2 && types.head == ConfigType.Client
+    val entityTypes = if (reverse) types.reverse else types
+    val sortedNames = (if (reverse && names.length == 2) names.reverse else names).iterator
+
+    def sanitizeName(entityType: String, name: String) = {
+      if (name.isEmpty)
+        ConfigEntityName.Default
+      else {
+        entityType match {
+          case ConfigType.User | ConfigType.Client => Sanitizer.sanitize(name)
+          case _ => throw new IllegalArgumentException("Invalid entity type " + entityType)
+        }
+      }
+    }
+
+    val entities = entityTypes.map(t => Entity(t, if (sortedNames.hasNext) Some(sanitizeName(t, sortedNames.next)) else None))
+    ConfigEntity(entities.head, if (entities.size > 1) Some(entities(1)) else None)
+  }
+
+  class ConfigCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+
+    val zkConnectOpt = parser.accepts("zookeeper", "DEPRECATED. The connection string for the zookeeper connection in the form host:port. " +
+            "Multiple URLS can be given to allow fail-over. Replaced by --bootstrap-server, REQUIRED unless --bootstrap-server is given.")
+            .withRequiredArg
+            .describedAs("urls")
+            .ofType(classOf[String])
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", "The Kafka server to connect to. " +
+      "This is required for describing and altering broker configs.")
+      .withRequiredArg
+      .describedAs("server to connect to")
+      .ofType(classOf[String])
+    val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client. " +
+      "This is used only with --bootstrap-server option for describing and altering broker configs.")
+      .withRequiredArg
+      .describedAs("command config property file")
+      .ofType(classOf[String])
+    val alterOpt = parser.accepts("alter", "Alter the configuration for the entity.")
+    val describeOpt = parser.accepts("describe", "List configs for the given entity.")
+    val allOpt = parser.accepts("all", "List all configs for the given entity (includes static configuration when the entity type is brokers)")
+
+    val entityType = parser.accepts("entity-type", "Type of entity (topics/clients/users/brokers/ha-clusters/ha-groups/ha-topics/broker-loggers)")
+            .withRequiredArg
+            .ofType(classOf[String])
+    val entityName = parser.accepts("entity-name", "Name of entity (topic name/client id/user principal name/ha-cluster id/ha-group id/ha-topic name/broker id)")
+            .withRequiredArg
+            .ofType(classOf[String])
+    val entityDefault = parser.accepts("entity-default", "Default entity name for clients/users/brokers (applies to corresponding entity type in command line)")
+
+    val nl = System.getProperty("line.separator")
+    val addConfig = parser.accepts("add-config", "Key Value pairs of configs to add. Square brackets can be used to group values which contain commas: 'k1=v1,k2=[v1,v2,v2],k3=v3'. The following is a list of valid configurations: " +
+            "For entity-type '" + ConfigType.Topic + "': " + LogConfig.configNames.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.Broker + "': " + DynamicConfig.Broker.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.User + "': " + DynamicConfig.User.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.Client + "': " + DynamicConfig.Client.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.HACluster + "': " + HAClusterConfig.configDef.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.HAGroup + "': " + HAGroupConfig.configDef.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            "For entity-type '" + ConfigType.HATopic + "': " + HATopicConfig.configDef.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) +
+            s"Entity types '${ConfigType.User}' and '${ConfigType.Client}' may be specified together to update config for clients of a specific user.")
+            .withRequiredArg
+            .ofType(classOf[String])
+    val deleteConfig = parser.accepts("delete-config", "config keys to remove 'k1,k2'")
+            .withRequiredArg
+            .ofType(classOf[String])
+            .withValuesSeparatedBy(',')
+    val forceOpt = parser.accepts("force", "Suppress console prompts")
+    val topic = parser.accepts("topic", "The topic's name.")
+      .withRequiredArg
+      .ofType(classOf[String])
+    val client = parser.accepts("client", "The client's ID.")
+      .withRequiredArg
+      .ofType(classOf[String])
+    val clientDefaults = parser.accepts("client-defaults", "The config defaults for all clients.")
+    val user = parser.accepts("user", "The user's principal name.")
+      .withRequiredArg
+      .ofType(classOf[String])
+    val userDefaults = parser.accepts("user-defaults", "The config defaults for all users.")
+    val broker = parser.accepts("broker", "The broker's ID.")
+      .withRequiredArg
+      .ofType(classOf[String])
+    val brokerDefaults = parser.accepts("broker-defaults", "The config defaults for all brokers.")
+    val brokerLogger = parser.accepts("broker-logger", "The broker's ID for its logger config.")
+      .withRequiredArg
+      .ofType(classOf[String])
+    val zkTlsConfigFile = parser.accepts("zk-tls-config-file",
+      "Identifies the file where ZooKeeper client TLS connectivity properties are defined.  Any properties other than " +
+        KafkaConfig.ZkSslConfigToSystemPropertyMap.keys.toList.sorted.mkString(", ") + " are ignored.")
+      .withRequiredArg().describedAs("ZooKeeper TLS configuration").ofType(classOf[String])
+    options = parser.parse(args : _*)
+
+    private val entityFlags = List((topic, ConfigType.Topic),
+      (client, ConfigType.Client),
+      (user, ConfigType.User),
+      (broker, ConfigType.Broker),
+      (brokerLogger, BrokerLoggerConfigType))
+
+    private val entityDefaultsFlags = List((clientDefaults, ConfigType.Client),
+      (userDefaults, ConfigType.User),
+      (brokerDefaults, ConfigType.Broker))
+
+    private[admin] def entityTypes(): List[String] = {
+      options.valuesOf(entityType).asScala.toList ++
+        (entityFlags ++ entityDefaultsFlags).filter(entity => options.has(entity._1)).map(_._2)
+    }
+
+    private[admin] def entityNames(): List[String] = {
+      val namesIterator = options.valuesOf(entityName).iterator
+      options.specs.asScala
+        .filter(spec => spec.options.contains("entity-name") || spec.options.contains("entity-default"))
+        .map(spec => if (spec.options.contains("entity-name")) namesIterator.next else "").toList ++
+      entityFlags
+        .filter(entity => options.has(entity._1))
+        .map(entity => options.valueOf(entity._1)) ++
+      entityDefaultsFlags
+        .filter(entity => options.has(entity._1))
+        .map(_ => "")
+    }
+
+    def checkArgs(): Unit = {
+      // should have exactly one action
+      val actions = Seq(alterOpt, describeOpt).count(options.has _)
+      if (actions != 1)
+        CommandLineUtils.printUsageAndDie(parser, "Command must include exactly one action: --describe, --alter")
+      // check required args
+      CommandLineUtils.checkInvalidArgs(parser, options, alterOpt, Set(describeOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, describeOpt, Set(alterOpt, addConfig, deleteConfig))
+
+      val entityTypeVals = entityTypes
+      if (entityTypeVals.size != entityTypeVals.distinct.size)
+        throw new IllegalArgumentException(s"Duplicate entity type(s) specified: ${entityTypeVals.diff(entityTypeVals.distinct).mkString(",")}")
+
+      val (allowedEntityTypes, connectOptString) = if (options.has(bootstrapServerOpt))
+        (BrokerSupportedConfigTypes, "--bootstrap-server")
+      else
+        (ConfigType.all, "--zookeeper")
+
+      entityTypeVals.foreach(entityTypeVal =>
+        if (!allowedEntityTypes.contains(entityTypeVal))
+          throw new IllegalArgumentException(s"Invalid entity type $entityTypeVal, the entity type must be one of ${allowedEntityTypes.mkString(",")} with the $connectOptString argument")
+      )
+      if (entityTypeVals.isEmpty)
+        throw new IllegalArgumentException("At least one entity type must be specified")
+      else if (entityTypeVals.size > 1 && !entityTypeVals.toSet.equals(Set(ConfigType.User, ConfigType.Client)))
+        throw new IllegalArgumentException(s"Only '${ConfigType.User}' and '${ConfigType.Client}' entity types may be specified together")
+
+      if ((options.has(entityName) || options.has(entityType) || options.has(entityDefault)) &&
+        (entityFlags ++ entityDefaultsFlags).exists(entity => options.has(entity._1)))
+        throw new IllegalArgumentException("--entity-{type,name,default} should not be used in conjunction with specific entity flags")
+
+      val hasEntityName = entityNames.exists(!_.isEmpty)
+      val hasEntityDefault = entityNames.exists(_.isEmpty)
+
+      if (!options.has(bootstrapServerOpt) && !options.has(zkConnectOpt))
+        throw new IllegalArgumentException("One of the required --bootstrap-server or --zookeeper arguments must be specified")
+      else if (options.has(bootstrapServerOpt) && options.has(zkConnectOpt))
+        throw new IllegalArgumentException("Only one of --bootstrap-server or --zookeeper must be specified")
+
+      if (options.has(allOpt) && options.has(zkConnectOpt)) {
+        throw new IllegalArgumentException(s"--bootstrap-server must be specified for --all")
+      }
+
+      if (hasEntityName && (entityTypeVals.contains(ConfigType.Broker) || entityTypeVals.contains(BrokerLoggerConfigType))) {
+        Seq(entityName, broker, brokerLogger).filter(options.has(_)).map(options.valueOf(_)).foreach { brokerId =>
+          try brokerId.toInt catch {
+            case _: NumberFormatException =>
+              throw new IllegalArgumentException(s"The entity name for ${entityTypeVals.head} must be a valid integer broker id, but it is: $brokerId")
+          }
+        }
+      }
+
+      if (entityTypeVals.contains(ConfigType.Client) || entityTypeVals.contains(ConfigType.User))
+        CommandLineUtils.checkRequiredArgs(parser, options, zkConnectOpt)
+
+      if (options.has(describeOpt) && entityTypeVals.contains(BrokerLoggerConfigType) && !hasEntityName)
+        throw new IllegalArgumentException(s"an entity name must be specified with --describe of ${entityTypeVals.mkString(",")}")
+
+      if (options.has(alterOpt)) {
+        if (entityTypeVals.contains(ConfigType.User) || entityTypeVals.contains(ConfigType.Client) || entityTypeVals.contains(ConfigType.Broker)) {
+          if (!hasEntityName && !hasEntityDefault)
+            throw new IllegalArgumentException("an entity-name or default entity must be specified with --alter of users, clients or brokers")
+        } else if (!hasEntityName)
+          throw new IllegalArgumentException(s"an entity name must be specified with --alter of ${entityTypeVals.mkString(",")}")
+
+        val isAddConfigPresent: Boolean = options.has(addConfig)
+        val isDeleteConfigPresent: Boolean = options.has(deleteConfig)
+        if (!isAddConfigPresent && !isDeleteConfigPresent)
+          throw new IllegalArgumentException("At least one of --add-config or --delete-config must be specified with --alter")
+      }
+    }
+  }
+}

core/src/main/scala/kafka/admin/DelegationTokenCommand.scala

@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.text.SimpleDateFormat
+import java.util
+import java.util.Base64
+
+import joptsimple.ArgumentAcceptingOptionSpec
+import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Exit, Logging}
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.clients.admin.{Admin, CreateDelegationTokenOptions, DescribeDelegationTokenOptions, ExpireDelegationTokenOptions, RenewDelegationTokenOptions}
+import org.apache.kafka.common.security.auth.KafkaPrincipal
+import org.apache.kafka.common.security.token.delegation.DelegationToken
+import org.apache.kafka.common.utils.{SecurityUtils, Utils}
+
+import scala.collection.JavaConverters._
+import scala.collection.Set
+
+/**
+ * A command to manage delegation token.
+ */
+object DelegationTokenCommand extends Logging {
+
+  def main(args: Array[String]): Unit = {
+    val opts = new DelegationTokenCommandOptions(args)
+
+    CommandLineUtils.printHelpAndExitIfNeeded(opts, "This tool helps to create, renew, expire, or describe delegation tokens.")
+
+    // should have exactly one action
+    val actions = Seq(opts.createOpt, opts.renewOpt, opts.expiryOpt, opts.describeOpt).count(opts.options.has _)
+    if(actions != 1)
+      CommandLineUtils.printUsageAndDie(opts.parser, "Command must include exactly one action: --create, --renew, --expire or --describe")
+
+    opts.checkArgs()
+
+    val adminClient = createAdminClient(opts)
+
+    var exitCode = 0
+    try {
+      if(opts.options.has(opts.createOpt))
+        createToken(adminClient, opts)
+      else if(opts.options.has(opts.renewOpt))
+        renewToken(adminClient, opts)
+      else if(opts.options.has(opts.expiryOpt))
+        expireToken(adminClient, opts)
+      else if(opts.options.has(opts.describeOpt))
+        describeToken(adminClient, opts)
+    } catch {
+      case e: Throwable =>
+        println("Error while executing delegation token command : " + e.getMessage)
+        error(Utils.stackTrace(e))
+        exitCode = 1
+    } finally {
+      adminClient.close()
+      Exit.exit(exitCode)
+    }
+  }
+
+  def createToken(adminClient: Admin, opts: DelegationTokenCommandOptions): DelegationToken = {
+    val renewerPrincipals = getPrincipals(opts, opts.renewPrincipalsOpt).getOrElse(new util.LinkedList[KafkaPrincipal]())
+    val maxLifeTimeMs = opts.options.valueOf(opts.maxLifeTimeOpt).longValue
+
+    println("Calling create token operation with renewers :" + renewerPrincipals +" , max-life-time-period :"+ maxLifeTimeMs)
+    val createDelegationTokenOptions = new CreateDelegationTokenOptions().maxlifeTimeMs(maxLifeTimeMs).renewers(renewerPrincipals)
+    val createResult = adminClient.createDelegationToken(createDelegationTokenOptions)
+    val token = createResult.delegationToken().get()
+    println("Created delegation token with tokenId : %s".format(token.tokenInfo.tokenId)); printToken(List(token))
+    token
+  }
+
+  def printToken(tokens: List[DelegationToken]): Unit = {
+    val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm")
+    print("\n%-15s %-30s %-15s %-25s %-15s %-15s %-15s".format("TOKENID", "HMAC", "OWNER", "RENEWERS", "ISSUEDATE", "EXPIRYDATE", "MAXDATE"))
+    for (token <- tokens) {
+      val tokenInfo = token.tokenInfo
+      print("\n%-15s %-30s %-15s %-25s %-15s %-15s %-15s".format(
+        tokenInfo.tokenId,
+        token.hmacAsBase64String,
+        tokenInfo.owner,
+        tokenInfo.renewersAsString,
+        dateFormat.format(tokenInfo.issueTimestamp),
+        dateFormat.format(tokenInfo.expiryTimestamp),
+        dateFormat.format(tokenInfo.maxTimestamp)))
+      println()
+    }
+  }
+
+  private def getPrincipals(opts: DelegationTokenCommandOptions, principalOptionSpec: ArgumentAcceptingOptionSpec[String]): Option[util.List[KafkaPrincipal]] = {
+    if (opts.options.has(principalOptionSpec))
+      Some(opts.options.valuesOf(principalOptionSpec).asScala.map(s => SecurityUtils.parseKafkaPrincipal(s.trim)).toList.asJava)
+    else
+      None
+  }
+
+  def renewToken(adminClient: Admin, opts: DelegationTokenCommandOptions): Long = {
+    val hmac = opts.options.valueOf(opts.hmacOpt)
+    val renewTimePeriodMs = opts.options.valueOf(opts.renewTimePeriodOpt).longValue()
+    println("Calling renew token operation with hmac :" + hmac +" , renew-time-period :"+ renewTimePeriodMs)
+    val renewResult = adminClient.renewDelegationToken(Base64.getDecoder.decode(hmac), new RenewDelegationTokenOptions().renewTimePeriodMs(renewTimePeriodMs))
+    val expiryTimeStamp = renewResult.expiryTimestamp().get()
+    val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm")
+    println("Completed renew operation. New expiry date : %s".format(dateFormat.format(expiryTimeStamp)))
+    expiryTimeStamp
+  }
+
+  def expireToken(adminClient: Admin, opts: DelegationTokenCommandOptions): Long = {
+    val hmac = opts.options.valueOf(opts.hmacOpt)
+    val expiryTimePeriodMs = opts.options.valueOf(opts.expiryTimePeriodOpt).longValue()
+    println("Calling expire token operation with hmac :" + hmac +" , expire-time-period : "+ expiryTimePeriodMs)
+    val expireResult = adminClient.expireDelegationToken(Base64.getDecoder.decode(hmac), new ExpireDelegationTokenOptions().expiryTimePeriodMs(expiryTimePeriodMs))
+    val expiryTimeStamp = expireResult.expiryTimestamp().get()
+    val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm")
+    println("Completed expire operation. New expiry date : %s".format(dateFormat.format(expiryTimeStamp)))
+    expiryTimeStamp
+  }
+
+  def describeToken(adminClient: Admin, opts: DelegationTokenCommandOptions): List[DelegationToken] = {
+    val ownerPrincipals = getPrincipals(opts, opts.ownerPrincipalsOpt)
+    if (ownerPrincipals.isEmpty)
+      println("Calling describe token operation for current user.")
+    else
+      println("Calling describe token operation for owners :" + ownerPrincipals.get)
+
+    val describeResult = adminClient.describeDelegationToken(new DescribeDelegationTokenOptions().owners(ownerPrincipals.orNull))
+    val tokens = describeResult.delegationTokens().get().asScala.toList
+    println("Total number of tokens : %s".format(tokens.size)); printToken(tokens)
+    tokens
+  }
+
+  private def createAdminClient(opts: DelegationTokenCommandOptions): Admin = {
+    val props = Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+    props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+    Admin.create(props)
+  }
+
+  class DelegationTokenCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    val BootstrapServerDoc = "REQUIRED: server(s) to use for bootstrapping."
+    val CommandConfigDoc = "REQUIRED: A property file containing configs to be passed to Admin Client. Token management" +
+      " operations are allowed in secure mode only. This config file is used to pass security related configs."
+
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", BootstrapServerDoc)
+                                   .withRequiredArg
+                                   .ofType(classOf[String])
+    val commandConfigOpt = parser.accepts("command-config", CommandConfigDoc)
+      .withRequiredArg
+      .ofType(classOf[String])
+
+    val createOpt = parser.accepts("create", "Create a new delegation token. Use --renewer-principal option to pass renewers principals.")
+    val renewOpt = parser.accepts("renew",  "Renew delegation token. Use --renew-time-period option to set renew time period.")
+    val expiryOpt = parser.accepts("expire", "Expire delegation token. Use --expiry-time-period option to expire the token.")
+    val describeOpt = parser.accepts("describe", "Describe delegation tokens for the given principals. Use --owner-principal to pass owner/renewer principals." +
+      " If --owner-principal option is not supplied, all the user owned tokens and tokens where user have Describe permission will be returned.")
+
+    val ownerPrincipalsOpt = parser.accepts("owner-principal", "owner is a kafka principal. It is should be in principalType:name format.")
+      .withOptionalArg()
+      .ofType(classOf[String])
+
+    val renewPrincipalsOpt = parser.accepts("renewer-principal", "renewer is a kafka principal. It is should be in principalType:name format.")
+      .withOptionalArg()
+      .ofType(classOf[String])
+
+    val maxLifeTimeOpt = parser.accepts("max-life-time-period", "Max life period for the token in milliseconds. If the value is -1," +
+      " then token max life time will default to a server side config value (delegation.token.max.lifetime.ms).")
+      .withOptionalArg()
+      .ofType(classOf[Long])
+
+    val renewTimePeriodOpt = parser.accepts("renew-time-period", "Renew time period in milliseconds. If the value is -1, then the" +
+      " renew time period will default to a server side config value (delegation.token.expiry.time.ms).")
+      .withOptionalArg()
+      .ofType(classOf[Long])
+
+    val expiryTimePeriodOpt = parser.accepts("expiry-time-period", "Expiry time period in milliseconds. If the value is -1, then the" +
+      " token will get invalidated immediately." )
+      .withOptionalArg()
+      .ofType(classOf[Long])
+
+    val hmacOpt = parser.accepts("hmac", "HMAC of the delegation token")
+      .withOptionalArg
+      .ofType(classOf[String])
+
+    options = parser.parse(args : _*)
+
+    def checkArgs(): Unit = {
+      // check required args
+      CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt, commandConfigOpt)
+
+      if (options.has(createOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, maxLifeTimeOpt)
+
+      if (options.has(renewOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, hmacOpt, renewTimePeriodOpt)
+
+      if (options.has(expiryOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, hmacOpt, expiryTimePeriodOpt)
+
+      // check invalid args
+      CommandLineUtils.checkInvalidArgs(parser, options, createOpt, Set(hmacOpt, renewTimePeriodOpt, expiryTimePeriodOpt, ownerPrincipalsOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, renewOpt, Set(renewPrincipalsOpt, maxLifeTimeOpt, expiryTimePeriodOpt, ownerPrincipalsOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, expiryOpt, Set(renewOpt, maxLifeTimeOpt, renewTimePeriodOpt, ownerPrincipalsOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, describeOpt, Set(renewTimePeriodOpt, maxLifeTimeOpt, hmacOpt, renewTimePeriodOpt, expiryTimePeriodOpt))
+    }
+  }
+}

core/src/main/scala/kafka/admin/DeleteRecordsCommand.scala

@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.io.PrintStream
+import java.util.Properties
+
+import kafka.common.AdminCommandFailedException
+import kafka.utils.json.JsonValue
+import kafka.utils.{CommandDefaultOptions, CommandLineUtils, CoreUtils, Json}
+import org.apache.kafka.clients.admin.{Admin, RecordsToDelete}
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.utils.Utils
+
+import scala.collection.JavaConverters._
+import scala.collection.Seq
+
+/**
+ * A command for delete records of the given partitions down to the specified offset.
+ */
+object DeleteRecordsCommand {
+
+  private[admin] val EarliestVersion = 1
+
+  def main(args: Array[String]): Unit = {
+    execute(args, System.out)
+  }
+
+  def parseOffsetJsonStringWithoutDedup(jsonData: String): Seq[(TopicPartition, Long)] = {
+    Json.parseFull(jsonData) match {
+      case Some(js) =>
+        val version = js.asJsonObject.get("version") match {
+          case Some(jsonValue) => jsonValue.to[Int]
+          case None => EarliestVersion
+        }
+        parseJsonData(version, js)
+      case None => throw new AdminOperationException("The input string is not a valid JSON")
+    }
+  }
+
+  def parseJsonData(version: Int, js: JsonValue): Seq[(TopicPartition, Long)] = {
+    version match {
+      case 1 =>
+        js.asJsonObject.get("partitions") match {
+          case Some(partitions) =>
+            partitions.asJsonArray.iterator.map(_.asJsonObject).map { partitionJs =>
+              val topic = partitionJs("topic").to[String]
+              val partition = partitionJs("partition").to[Int]
+              val offset = partitionJs("offset").to[Long]
+              new TopicPartition(topic, partition) -> offset
+            }.toBuffer
+          case _ => throw new AdminOperationException("Missing partitions field");
+        }
+      case _ => throw new AdminOperationException(s"Not supported version field value $version")
+    }
+  }
+
+  def execute(args: Array[String], out: PrintStream): Unit = {
+    val opts = new DeleteRecordsCommandOptions(args)
+    val adminClient = createAdminClient(opts)
+    val offsetJsonFile =  opts.options.valueOf(opts.offsetJsonFileOpt)
+    val offsetJsonString = Utils.readFileAsString(offsetJsonFile)
+    val offsetSeq = parseOffsetJsonStringWithoutDedup(offsetJsonString)
+
+    val duplicatePartitions = CoreUtils.duplicates(offsetSeq.map { case (tp, _) => tp })
+    if (duplicatePartitions.nonEmpty)
+      throw new AdminCommandFailedException("Offset json file contains duplicate topic partitions: %s".format(duplicatePartitions.mkString(",")))
+
+    val recordsToDelete = offsetSeq.map { case (topicPartition, offset) =>
+      (topicPartition, RecordsToDelete.beforeOffset(offset))
+    }.toMap.asJava
+
+    out.println("Executing records delete operation")
+    val deleteRecordsResult = adminClient.deleteRecords(recordsToDelete)
+    out.println("Records delete operation completed:")
+
+    deleteRecordsResult.lowWatermarks.asScala.foreach { case (tp, partitionResult) => {
+      try out.println(s"partition: $tp\tlow_watermark: ${partitionResult.get.lowWatermark}")
+      catch {
+        case e: Exception => out.println(s"partition: $tp\terror: ${e.getMessage}")
+      }
+    }}
+
+    adminClient.close()
+  }
+
+  private def createAdminClient(opts: DeleteRecordsCommandOptions): Admin = {
+    val props = if (opts.options.has(opts.commandConfigOpt))
+      Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+    else
+      new Properties()
+    props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+    Admin.create(props)
+  }
+
+  class DeleteRecordsCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    val BootstrapServerDoc = "REQUIRED: The server to connect to."
+    val offsetJsonFileDoc = "REQUIRED: The JSON file with offset per partition. The format to use is:\n" +
+                                 "{\"partitions\":\n  [{\"topic\": \"foo\", \"partition\": 1, \"offset\": 1}],\n \"version\":1\n}"
+    val CommandConfigDoc = "A property file containing configs to be passed to Admin Client."
+
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", BootstrapServerDoc)
+                                   .withRequiredArg
+                                   .describedAs("server(s) to use for bootstrapping")
+                                   .ofType(classOf[String])
+    val offsetJsonFileOpt = parser.accepts("offset-json-file", offsetJsonFileDoc)
+                                   .withRequiredArg
+                                   .describedAs("Offset json file path")
+                                   .ofType(classOf[String])
+    val commandConfigOpt = parser.accepts("command-config", CommandConfigDoc)
+                                   .withRequiredArg
+                                   .describedAs("command config property file path")
+                                   .ofType(classOf[String])
+
+    options = parser.parse(args : _*)
+
+    CommandLineUtils.printHelpAndExitIfNeeded(this, "This tool helps to delete records of the given partitions down to the specified offset.")
+
+    CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt, offsetJsonFileOpt)
+  }
+}

core/src/main/scala/kafka/admin/DiskLoadProtectorCommand.scala

@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.io.FileReader
+import java.util.Properties
+
+import com.didichuxing.datachannel.kafka.server.DiskLoadProtector
+import joptsimple.OptionParser
+import kafka.server.ConfigType
+import kafka.utils.{CommandLineUtils, Logging}
+import kafka.zk.{AdminZkClient, KafkaZkClient}
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.utils.{Time, Utils}
+
+import scala.collection._
+
+
+object DiskLoadProtectorCommand extends Logging {
+
+  def main(args: Array[String]): Unit = {
+
+    val opts = validateAndParseArgs(args)
+    val zkConnect = opts.options.valueOf(opts.zkConnectOpt)
+    val time = Time.SYSTEM
+    val zkClient = KafkaZkClient(zkConnect, JaasUtils.isZkSaslEnabled, 30000, 30000, Int.MaxValue, time)
+    val adminZkClient = new AdminZkClient(zkClient)
+    try {
+      var brokerId = "";
+      if (opts.options.has(opts.configBrokerIdOpt)) {
+        brokerId = opts.options.valueOf(opts.configBrokerIdOpt);
+      }
+
+      if (opts.options.has(opts.describeOpt)) {
+        val diskLoadProtector = new DiskLoadProtector()
+        var finalConfig = diskLoadProtector.getConfig;
+        if (brokerId.nonEmpty) {
+          val config = readConfig(adminZkClient, brokerId)
+          if (config == null) {
+            config.keySet().forEach( key  => finalConfig.put(key, config.get(key)))
+          }
+          print("broker %s config is:\n".format(brokerId))
+          printConfig(finalConfig);
+        } else {
+          print("all broker's config is:\n".format(brokerId))
+          val config = readConfig(adminZkClient, brokerId);
+          if (config == null) {
+            config.keySet().forEach( key  => finalConfig.put(key, config.get(key)))
+          }
+          printConfig(finalConfig)
+          println();
+
+          finalConfig = diskLoadProtector.getConfig;
+          readAllConfig(adminZkClient).foreach{
+            case (brokerId, props) =>
+              print("broker %s config is:\n".format(brokerId))
+              props.keySet().forEach( key  => finalConfig.put(key, config.get(key)))
+              printConfig(finalConfig);
+              println();
+          }
+        }
+      } else {
+        val configFileName = opts.options.valueOf(opts.configFileOpt);
+        val properties = new Properties();
+        properties.load(new FileReader(configFileName));
+        changeEntityConfig(adminZkClient, ConfigType.DiskLoadProtector, brokerId, properties);
+      }
+    } catch {
+      case e: Throwable =>
+
+        println("Partitions reassignment failed due to " + e.getMessage)
+        println(Utils.stackTrace(e))
+    } finally zkClient.close()
+  }
+
+  def validateAndParseArgs(args: Array[String]): CommandOptions = {
+    val opts = new CommandOptions(args);
+    if (args.length == 0)
+      CommandLineUtils.printUsageAndDie(opts.parser, "This command get and set configuration for disk load protector.")
+
+    CommandLineUtils.checkRequiredArgs(opts.parser, opts.options, opts.zkConnectOpt)
+    opts
+  }
+
+  class CommandOptions(args: Array[String]) {
+    val parser = new OptionParser
+
+    val zkConnectOpt = parser.accepts("zookeeper", "REQUIRED: The connection string for the zookeeper connection in the " +
+      "form host:port. Multiple URLS can be given to allow fail-over.")
+      .withRequiredArg
+      .describedAs("urls")
+      .ofType(classOf[String])
+    val configFileOpt = parser.accepts("config", "The json file with the disk load protector configuration")
+      .withOptionalArg()
+      .describedAs("manual assignment file path")
+      .ofType(classOf[String])
+    val configBrokerIdOpt = parser.accepts("broker", "The special broker to be set configuration")
+      .withOptionalArg()
+      .describedAs("broker id")
+      .ofType(classOf[String])
+    val describeOpt = parser.accepts("describe", "describe the configuration")
+      .withOptionalArg()
+      .describedAs("describe")
+    val options = parser.parse(args: _*)
+  }
+
+  private def changeEntityConfig(zkUtils: AdminZkClient, rootEntityType: String, fullSanitizedEntityName: String, config: Properties) = {
+    zkUtils.changeEntityConfig(rootEntityType, fullSanitizedEntityName, config)
+  }
+
+  private def readConfig(zkUtils: AdminZkClient, brokerId: String): Properties = {
+    zkUtils.fetchEntityConfig(ConfigType.DiskLoadProtector, brokerId)
+  }
+
+  private def readAllConfig(zkUtils: AdminZkClient): Map[String, Properties] = {
+    zkUtils.fetchAllEntityConfigs(ConfigType.DiskLoadProtector)
+  }
+
+  private def printConfig(properties: Properties): Unit = {
+    properties.keySet().forEach(
+      key  => print("\t%s=%s\n".format(key, properties.get(key)))
+    )
+  }
+}

core/src/main/scala/kafka/admin/KafkaExMetricsCommand.scala

@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.io.FileReader
+import java.util.Properties
+
+import joptsimple.OptionParser
+import kafka.server.ConfigType
+import kafka.utils.{CommandLineUtils, Logging}
+import kafka.zk.{AdminZkClient, KafkaZkClient}
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.utils.{Time, Utils}
+
+import scala.collection._
+
+
+object KafkaExMetricsCommand extends Logging {
+
+  def main(args: Array[String]): Unit = {
+
+    val opts = validateAndParseArgs(args)
+    val zkConnect = opts.options.valueOf(opts.zkConnectOpt)
+    val time = Time.SYSTEM
+    val zkClient = KafkaZkClient(zkConnect, JaasUtils.isZkSaslEnabled, 30000, 30000, Int.MaxValue, time)
+    val adminZkClient = new AdminZkClient(zkClient)
+    try {
+      var brokerId = "";
+      if (opts.options.has(opts.configBrokerIdOpt)) {
+        brokerId = opts.options.valueOf(opts.configBrokerIdOpt);
+      }
+
+      if (opts.options.has(opts.describeOpt)) {
+        if (brokerId.nonEmpty) {
+          val config = readConfig(adminZkClient, brokerId)
+          print("broker %s config is:\n".format(brokerId))
+          printConfig(config);
+        } else {
+          print("all broker's config is:\n")
+          val config = readConfig(adminZkClient, "");
+          printConfig(config)
+          println();
+
+          readAllConfig(adminZkClient).foreach{
+            case (brokerId, props) =>
+              print("broker %s config is:\n".format(brokerId))
+              printConfig(props)
+              println();
+          }
+        }
+      } else {
+        val configFileName = opts.options.valueOf(opts.configFileOpt);
+        val properties = new Properties();
+        properties.load(new FileReader(configFileName));
+        changeEntityConfig(adminZkClient, ConfigType.KafkaExMetrics, brokerId, properties);
+      }
+    } catch {
+      case e: Throwable =>
+
+        println("Partitions reassignment failed due to " + e.getMessage)
+        println(Utils.stackTrace(e))
+    } finally zkClient.close()
+  }
+
+  def validateAndParseArgs(args: Array[String]): CommandOptions = {
+    val opts = new CommandOptions(args);
+    if (args.length == 0)
+      CommandLineUtils.printUsageAndDie(opts.parser, "This command get and set configuration for disk load protector.")
+
+    CommandLineUtils.checkRequiredArgs(opts.parser, opts.options, opts.zkConnectOpt)
+    opts
+  }
+
+  class CommandOptions(args: Array[String]) {
+    val parser = new OptionParser
+
+    val zkConnectOpt = parser.accepts("zookeeper", "REQUIRED: The connection string for the zookeeper connection in the " +
+      "form host:port. Multiple URLS can be given to allow fail-over.")
+      .withRequiredArg
+      .describedAs("urls")
+      .ofType(classOf[String])
+    val configFileOpt = parser.accepts("config", "The json file with the disk load protector configuration")
+      .withOptionalArg()
+      .describedAs("manual assignment file path")
+      .ofType(classOf[String])
+    val configBrokerIdOpt = parser.accepts("broker", "The special broker to be set configuration")
+      .withOptionalArg()
+      .describedAs("broker id")
+      .ofType(classOf[String])
+    val describeOpt = parser.accepts("describe", "describe the configuration")
+      .withOptionalArg()
+      .describedAs("describe")
+    val options = parser.parse(args: _*)
+  }
+
+  private def changeEntityConfig(zkUtils: AdminZkClient, rootEntityType: String, fullSanitizedEntityName: String, config: Properties) = {
+    zkUtils.changeEntityConfig(rootEntityType, fullSanitizedEntityName, config)
+  }
+
+  private def readConfig(zkUtils: AdminZkClient, brokerId: String): Properties = {
+    zkUtils.fetchEntityConfig(ConfigType.KafkaExMetrics, brokerId)
+  }
+
+  private def readAllConfig(zkUtils: AdminZkClient): Map[String, Properties] = {
+    zkUtils.fetchAllEntityConfigs(ConfigType.KafkaExMetrics)
+  }
+
+  private def printConfig(properties: Properties): Unit = {
+    properties.keySet().forEach(
+      key  => print("\t%s=%s\n".format(key, properties.get(key)))
+    )
+  }
+}

core/src/main/scala/kafka/admin/LeaderElectionCommand.scala

@@ -0,0 +1,288 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.admin
+
+import java.util.Properties
+import java.util.concurrent.ExecutionException
+import joptsimple.util.EnumConverter
+import kafka.common.AdminCommandFailedException
+import kafka.utils.CommandDefaultOptions
+import kafka.utils.CommandLineUtils
+import kafka.utils.CoreUtils
+import kafka.utils.Json
+import kafka.utils.Logging
+import org.apache.kafka.clients.admin.{Admin, AdminClientConfig}
+import org.apache.kafka.common.ElectionType
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.errors.ClusterAuthorizationException
+import org.apache.kafka.common.errors.ElectionNotNeededException
+import org.apache.kafka.common.errors.TimeoutException
+import org.apache.kafka.common.utils.Utils
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.concurrent.duration._
+
+object LeaderElectionCommand extends Logging {
+  def main(args: Array[String]): Unit = {
+    run(args, 30.second)
+  }
+
+  def run(args: Array[String], timeout: Duration): Unit = {
+    val commandOptions = new LeaderElectionCommandOptions(args)
+    CommandLineUtils.printHelpAndExitIfNeeded(
+      commandOptions,
+      "This tool attempts to elect a new leader for a set of topic partitions. The type of elections supported are preferred replicas and unclean replicas."
+    )
+
+    validate(commandOptions)
+
+    val electionType = commandOptions.options.valueOf(commandOptions.electionType)
+
+    val jsonFileTopicPartitions = Option(commandOptions.options.valueOf(commandOptions.pathToJsonFile)).map { path  =>
+      parseReplicaElectionData(Utils.readFileAsString(path))
+    }
+
+    val singleTopicPartition = (
+      Option(commandOptions.options.valueOf(commandOptions.topic)),
+      Option(commandOptions.options.valueOf(commandOptions.partition))
+    ) match {
+      case (Some(topic), Some(partition)) => Some(Set(new TopicPartition(topic, partition)))
+      case _ => None
+    }
+
+    /* Note: No need to look at --all-topic-partitions as we want this to be None if it is use.
+     * The validate function should be checking that this option is required if the --topic and --path-to-json-file
+     * are not specified.
+     */
+    val topicPartitions = jsonFileTopicPartitions.orElse(singleTopicPartition)
+
+    val adminClient = {
+      val props = Option(commandOptions.options.valueOf(commandOptions.adminClientConfig)).map { config =>
+        Utils.loadProps(config)
+      }.getOrElse(new Properties())
+
+      props.setProperty(
+        AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,
+        commandOptions.options.valueOf(commandOptions.bootstrapServer)
+      )
+      props.setProperty(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, timeout.toMillis.toString)
+      props.setProperty(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, (timeout.toMillis / 2).toString)
+
+      Admin.create(props)
+    }
+
+    try {
+      electLeaders(adminClient, electionType, topicPartitions)
+    } finally {
+      adminClient.close()
+    }
+  }
+
+  private[this] def parseReplicaElectionData(jsonString: String): Set[TopicPartition] = {
+    Json.parseFull(jsonString) match {
+      case Some(js) =>
+        js.asJsonObject.get("partitions") match {
+          case Some(partitionsList) =>
+            val partitionsRaw = partitionsList.asJsonArray.iterator.map(_.asJsonObject)
+            val partitions = partitionsRaw.map { p =>
+              val topic = p("topic").to[String]
+              val partition = p("partition").to[Int]
+              new TopicPartition(topic, partition)
+            }.toBuffer
+            val duplicatePartitions = CoreUtils.duplicates(partitions)
+            if (duplicatePartitions.nonEmpty) {
+              throw new AdminOperationException(
+                s"Replica election data contains duplicate partitions: ${duplicatePartitions.mkString(",")}"
+              )
+            }
+            partitions.toSet
+          case None => throw new AdminOperationException("Replica election data is missing \"partitions\" field")
+        }
+      case None => throw new AdminOperationException("Replica election data is empty")
+    }
+  }
+
+  private[this] def electLeaders(
+    client: Admin,
+    electionType: ElectionType,
+    topicPartitions: Option[Set[TopicPartition]]
+  ): Unit = {
+    val electionResults = try {
+      val partitions = topicPartitions.map(_.asJava).orNull
+      debug(s"Calling AdminClient.electLeaders($electionType, $partitions)")
+      client.electLeaders(electionType, partitions).partitions.get.asScala
+    } catch {
+      case e: ExecutionException =>
+        e.getCause match {
+          case cause: TimeoutException =>
+            val message = "Timeout waiting for election results"
+            println(message)
+            throw new AdminCommandFailedException(message, cause)
+          case cause: ClusterAuthorizationException =>
+            val message = "Not authorized to perform leader election"
+            println(message)
+            throw new AdminCommandFailedException(message, cause)
+          case _ =>
+            throw e
+        }
+      case e: Throwable =>
+        println("Error while making request")
+        throw e
+    }
+
+    val succeeded = mutable.Set.empty[TopicPartition]
+    val noop = mutable.Set.empty[TopicPartition]
+    val failed = mutable.Map.empty[TopicPartition, Throwable]
+
+    electionResults.foreach[Unit] { case (topicPartition, error) =>
+      if (error.isPresent) {
+        error.get match {
+          case _: ElectionNotNeededException => noop += topicPartition
+          case _ => failed += topicPartition -> error.get
+        }
+      } else {
+        succeeded += topicPartition
+      }
+    }
+
+    if (succeeded.nonEmpty) {
+      val partitions = succeeded.mkString(", ")
+      println(s"Successfully completed leader election ($electionType) for partitions $partitions")
+    }
+
+    if (noop.nonEmpty) {
+      val partitions = succeeded.mkString(", ")
+      println(s"Valid replica already elected for partitions $partitions")
+    }
+
+    if (failed.nonEmpty) {
+      val rootException = new AdminCommandFailedException(s"${failed.size} replica(s) could not be elected")
+      failed.foreach { case (topicPartition, exception) =>
+        println(s"Error completing leader election ($electionType) for partition: $topicPartition: $exception")
+        rootException.addSuppressed(exception)
+      }
+      throw rootException
+    }
+  }
+
+  private[this] def validate(commandOptions: LeaderElectionCommandOptions): Unit = {
+    // required options: --bootstrap-server and --election-type
+    var missingOptions = List.empty[String]
+    if (!commandOptions.options.has(commandOptions.bootstrapServer)) {
+      missingOptions = commandOptions.bootstrapServer.options().get(0) :: missingOptions
+    }
+
+    if (!commandOptions.options.has(commandOptions.electionType)) {
+      missingOptions = commandOptions.electionType.options().get(0) :: missingOptions
+    }
+
+    if (missingOptions.nonEmpty) {
+      throw new AdminCommandFailedException(s"Missing required option(s): ${missingOptions.mkString(", ")}")
+    }
+
+    // One and only one is required: --topic, --all-topic-partitions or --path-to-json-file
+    val mutuallyExclusiveOptions = Seq(
+      commandOptions.topic,
+      commandOptions.allTopicPartitions,
+      commandOptions.pathToJsonFile
+    )
+
+    mutuallyExclusiveOptions.count(commandOptions.options.has) match {
+      case 1 => // This is the only correct configuration, don't throw an exception
+      case _ =>
+        throw new AdminCommandFailedException(
+          "One and only one of the following options is required: " +
+          s"${mutuallyExclusiveOptions.map(_.options.get(0)).mkString(", ")}"
+        )
+    }
+
+    // --partition if and only if --topic is used
+    (
+      commandOptions.options.has(commandOptions.topic),
+      commandOptions.options.has(commandOptions.partition)
+    ) match {
+      case (true, false) =>
+        throw new AdminCommandFailedException(
+          s"Missing required option(s): ${commandOptions.partition.options.get(0)}"
+        )
+      case (false, true) =>
+        throw new AdminCommandFailedException(
+          s"Option ${commandOptions.partition.options.get(0)} is only allowed if " +
+          s"${commandOptions.topic.options.get(0)} is used"
+        )
+      case _ => // Ignore; we have a valid configuration
+    }
+  }
+}
+
+private final class LeaderElectionCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+  val bootstrapServer = parser
+    .accepts(
+      "bootstrap-server",
+      "A hostname and port for the broker to connect to, in the form host:port. Multiple comma separated URLs can be given. REQUIRED.")
+    .withRequiredArg
+    .describedAs("host:port")
+    .ofType(classOf[String])
+  val adminClientConfig = parser
+    .accepts(
+      "admin.config",
+      "Configuration properties files to pass to the admin client")
+    .withRequiredArg
+    .describedAs("config file")
+    .ofType(classOf[String])
+
+  val pathToJsonFile = parser
+    .accepts(
+      "path-to-json-file",
+      "The JSON file with the list  of partition for which leader elections should be performed. This is an example format. \n{\"partitions\":\n\t[{\"topic\": \"foo\", \"partition\": 1},\n\t {\"topic\": \"foobar\", \"partition\": 2}]\n}\nNot allowed if --all-topic-partitions or --topic flags are specified.")
+    .withRequiredArg
+    .describedAs("Path to JSON file")
+    .ofType(classOf[String])
+
+  val topic = parser
+    .accepts(
+      "topic",
+      "Name of topic for which to perform an election. Not allowed if --path-to-json-file or --all-topic-partitions is specified.")
+    .withRequiredArg
+    .describedAs("topic name")
+    .ofType(classOf[String])
+
+  val partition = parser
+    .accepts(
+      "partition",
+      "Partition id for which to perform an election. REQUIRED if --topic is specified.")
+    .withRequiredArg
+    .describedAs("partition id")
+    .ofType(classOf[Integer])
+
+  val allTopicPartitions = parser
+    .accepts(
+      "all-topic-partitions",
+      "Perform election on all of the eligible topic partitions based on the type of election (see the --election-type flag). Not allowed if --topic or --path-to-json-file is specified.")
+
+  val electionType = parser
+    .accepts(
+      "election-type",
+      "Type of election to attempt. Possible values are \"preferred\" for preferred leader election or \"unclean\" for unclean leader election. If preferred election is selection, the election is only performed if the current leader is not the preferred leader for the topic partition. If unclean election is selected, the election is only performed if there are no leader for the topic partition. REQUIRED.")
+    .withRequiredArg
+    .describedAs("election type")
+    .withValuesConvertedBy(ElectionTypeConverter)
+
+  options = parser.parse(args: _*)
+}
+
+final object ElectionTypeConverter extends EnumConverter[ElectionType](classOf[ElectionType]) { }

core/src/main/scala/kafka/admin/LogDirsCommand.scala

@@ -0,0 +1,123 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.admin
+
+import java.io.PrintStream
+import java.util.Properties
+
+import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Json}
+import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, DescribeLogDirsResult}
+import org.apache.kafka.common.requests.DescribeLogDirsResponse.LogDirInfo
+import org.apache.kafka.common.utils.Utils
+
+import scala.collection.JavaConverters._
+import scala.collection.Map
+
+/**
+  * A command for querying log directory usage on the specified brokers
+  */
+object LogDirsCommand {
+
+    def main(args: Array[String]): Unit = {
+        describe(args, System.out)
+    }
+
+    def describe(args: Array[String], out: PrintStream): Unit = {
+        val opts = new LogDirsCommandOptions(args)
+        val adminClient = createAdminClient(opts)
+        val topicList = opts.options.valueOf(opts.topicListOpt).split(",").filter(!_.isEmpty)
+        val brokerList = Option(opts.options.valueOf(opts.brokerListOpt)) match {
+            case Some(brokerListStr) => brokerListStr.split(',').filter(!_.isEmpty).map(_.toInt)
+            case None => adminClient.describeCluster().nodes().get().asScala.map(_.id()).toArray
+        }
+
+        out.println("Querying brokers for log directories information")
+        val describeLogDirsResult: DescribeLogDirsResult = adminClient.describeLogDirs(brokerList.map(Integer.valueOf).toSeq.asJava)
+        val logDirInfosByBroker = describeLogDirsResult.all.get().asScala.mapValues(_.asScala).toMap
+
+        out.println(s"Received log directory information from brokers ${brokerList.mkString(",")}")
+        out.println(formatAsJson(logDirInfosByBroker, topicList.toSet))
+        adminClient.close()
+    }
+
+    private def formatAsJson(logDirInfosByBroker: Map[Integer, Map[String, LogDirInfo]], topicSet: Set[String]): String = {
+        Json.encodeAsString(Map(
+            "version" -> 1,
+            "brokers" -> logDirInfosByBroker.map { case (broker, logDirInfos) =>
+                Map(
+                    "broker" -> broker,
+                    "logDirs" -> logDirInfos.map { case (logDir, logDirInfo) =>
+                        Map(
+                            "logDir" -> logDir,
+                            "error" -> logDirInfo.error.exceptionName(),
+                            "partitions" -> logDirInfo.replicaInfos.asScala.filter { case (topicPartition, _) =>
+                                topicSet.isEmpty || topicSet.contains(topicPartition.topic)
+                            }.map { case (topicPartition, replicaInfo) =>
+                                Map(
+                                    "partition" -> topicPartition.toString,
+                                    "size" -> replicaInfo.size,
+                                    "offsetLag" -> replicaInfo.offsetLag,
+                                    "isFuture" -> replicaInfo.isFuture
+                                ).asJava
+                            }.asJava
+                        ).asJava
+                    }.asJava
+                ).asJava
+            }.asJava
+        ).asJava)
+    }
+
+    private def createAdminClient(opts: LogDirsCommandOptions): Admin = {
+        val props = if (opts.options.has(opts.commandConfigOpt))
+            Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+        else
+            new Properties()
+        props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+        props.putIfAbsent(AdminClientConfig.CLIENT_ID_CONFIG, "log-dirs-tool")
+        Admin.create(props)
+    }
+
+    class LogDirsCommandOptions(args: Array[String]) extends CommandDefaultOptions(args){
+        val bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: the server(s) to use for bootstrapping")
+          .withRequiredArg
+          .describedAs("The server(s) to use for bootstrapping")
+          .ofType(classOf[String])
+        val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client.")
+          .withRequiredArg
+          .describedAs("Admin client property file")
+          .ofType(classOf[String])
+        val describeOpt = parser.accepts("describe", "Describe the specified log directories on the specified brokers.")
+        val topicListOpt = parser.accepts("topic-list", "The list of topics to be queried in the form \"topic1,topic2,topic3\". " +
+          "All topics will be queried if no topic list is specified")
+          .withRequiredArg
+          .describedAs("Topic list")
+          .defaultsTo("")
+          .ofType(classOf[String])
+        val brokerListOpt = parser.accepts("broker-list", "The list of brokers to be queried in the form \"0,1,2\". " +
+          "All brokers in the cluster will be queried if no broker list is specified")
+          .withRequiredArg
+          .describedAs("Broker list")
+          .ofType(classOf[String])
+
+        options = parser.parse(args : _*)
+
+        CommandLineUtils.printHelpAndExitIfNeeded(this, "This tool helps to query log directory usage on the specified brokers.")
+
+        CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt, describeOpt)
+    }
+}

core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala

@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.admin
+
+import collection.JavaConverters._
+import collection._
+import java.util.Properties
+import java.util.concurrent.ExecutionException
+
+import joptsimple.OptionSpecBuilder
+import kafka.common.AdminCommandFailedException
+import kafka.utils._
+import kafka.zk.KafkaZkClient
+import org.apache.kafka.clients.admin.{Admin, AdminClientConfig}
+import org.apache.kafka.common.ElectionType
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.errors.ClusterAuthorizationException
+import org.apache.kafka.common.errors.ElectionNotNeededException
+import org.apache.kafka.common.errors.TimeoutException
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.utils.Time
+import org.apache.kafka.common.utils.Utils
+import org.apache.zookeeper.KeeperException.NodeExistsException
+
+object PreferredReplicaLeaderElectionCommand extends Logging {
+
+  def main(args: Array[String]): Unit = {
+    val timeout = 30000
+    run(args, timeout)
+  }
+
+  def run(args: Array[String], timeout: Int = 30000): Unit = {
+    println("This tool is deprecated. Please use kafka-leader-election tool. Tracking issue: KAFKA-8405")
+    val commandOpts = new PreferredReplicaLeaderElectionCommandOptions(args)
+    CommandLineUtils.printHelpAndExitIfNeeded(commandOpts, "This tool helps to causes leadership for each partition to be transferred back to the 'preferred replica'," +
+      " it can be used to balance leadership among the servers.")
+
+    CommandLineUtils.checkRequiredArgs(commandOpts.parser, commandOpts.options)
+
+    if (commandOpts.options.has(commandOpts.bootstrapServerOpt) == commandOpts.options.has(commandOpts.zkConnectOpt)) {
+      CommandLineUtils.printUsageAndDie(commandOpts.parser, s"Exactly one of '${commandOpts.bootstrapServerOpt}' or '${commandOpts.zkConnectOpt}' must be provided")
+    }
+
+    val partitionsForPreferredReplicaElection =
+    if (commandOpts.options.has(commandOpts.jsonFileOpt))
+      Some(parsePreferredReplicaElectionData(Utils.readFileAsString(commandOpts.options.valueOf(commandOpts.jsonFileOpt))))
+    else
+      None
+
+    val preferredReplicaElectionCommand = if (commandOpts.options.has(commandOpts.zkConnectOpt)) {
+      println(s"Warning: --zookeeper is deprecated and will be removed in a future version of Kafka.")
+      println(s"Use --bootstrap-server instead to specify a broker to connect to.")
+      new ZkCommand(commandOpts.options.valueOf(commandOpts.zkConnectOpt),
+              JaasUtils.isZkSaslEnabled,
+              timeout)
+    } else {
+        val adminProps = if (commandOpts.options.has(commandOpts.adminClientConfigOpt))
+          Utils.loadProps(commandOpts.options.valueOf(commandOpts.adminClientConfigOpt))
+        else
+          new Properties()
+        adminProps.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, commandOpts.options.valueOf(commandOpts.bootstrapServerOpt))
+        adminProps.setProperty(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, timeout.toString)
+        adminProps.setProperty(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, (timeout * 2).toString)
+        new AdminClientCommand(adminProps)
+    }
+
+    try {
+      preferredReplicaElectionCommand.electPreferredLeaders(partitionsForPreferredReplicaElection)
+    } finally {
+      preferredReplicaElectionCommand.close()
+    }
+  }
+
+  def parsePreferredReplicaElectionData(jsonString: String): immutable.Set[TopicPartition] = {
+    Json.parseFull(jsonString) match {
+      case Some(js) =>
+        js.asJsonObject.get("partitions") match {
+          case Some(partitionsList) =>
+            val partitionsRaw = partitionsList.asJsonArray.iterator.map(_.asJsonObject)
+            val partitions = partitionsRaw.map { p =>
+              val topic = p("topic").to[String]
+              val partition = p("partition").to[Int]
+              new TopicPartition(topic, partition)
+            }.toBuffer
+            val duplicatePartitions = CoreUtils.duplicates(partitions)
+            if (duplicatePartitions.nonEmpty)
+              throw new AdminOperationException("Preferred replica election data contains duplicate partitions: %s".format(duplicatePartitions.mkString(",")))
+            partitions.toSet
+          case None => throw new AdminOperationException("Preferred replica election data is empty")
+        }
+      case None => throw new AdminOperationException("Preferred replica election data is empty")
+    }
+  }
+
+  def writePreferredReplicaElectionData(zkClient: KafkaZkClient,
+                                        partitionsUndergoingPreferredReplicaElection: Set[TopicPartition]): Unit = {
+    try {
+      zkClient.createPreferredReplicaElection(partitionsUndergoingPreferredReplicaElection.toSet)
+      println("Created preferred replica election path with %s".format(partitionsUndergoingPreferredReplicaElection.mkString(",")))
+    } catch {
+      case _: NodeExistsException =>
+        throw new AdminOperationException("Preferred replica leader election currently in progress for " +
+          "%s. Aborting operation".format(zkClient.getPreferredReplicaElection.mkString(",")))
+      case e2: Throwable => throw new AdminOperationException(e2.toString)
+    }
+  }
+
+  class PreferredReplicaLeaderElectionCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    val jsonFileOpt = parser.accepts("path-to-json-file", "The JSON file with the list of partitions " +
+      "for which preferred replica leader election should be done, in the following format - \n" +
+      "{\"partitions\":\n\t[{\"topic\": \"foo\", \"partition\": 1},\n\t {\"topic\": \"foobar\", \"partition\": 2}]\n}\n" +
+      "Defaults to all existing partitions")
+      .withRequiredArg
+      .describedAs("list of partitions for which preferred replica leader election needs to be triggered")
+      .ofType(classOf[String])
+
+    private val zookeeperOptBuilder: OptionSpecBuilder = parser.accepts("zookeeper",
+      "DEPRECATED. The connection string for the zookeeper connection in the " +
+      "form host:port. Multiple URLS can be given to allow fail-over. " +
+      "Replaced by --bootstrap-server, REQUIRED unless --bootstrap-server is given.")
+    private val bootstrapOptBuilder: OptionSpecBuilder = parser.accepts("bootstrap-server",
+      "A hostname and port for the broker to connect to, " +
+      "in the form host:port. Multiple comma-separated URLs can be given. REQUIRED unless --zookeeper is given.")
+    parser.mutuallyExclusive(zookeeperOptBuilder, bootstrapOptBuilder)
+    val bootstrapServerOpt = bootstrapOptBuilder
+      .withRequiredArg
+      .describedAs("host:port")
+      .ofType(classOf[String])
+    val zkConnectOpt = zookeeperOptBuilder
+      .withRequiredArg
+      .describedAs("urls")
+      .ofType(classOf[String])
+
+    val adminClientConfigOpt = parser.accepts("admin.config",
+      "Admin client config properties file to pass to the admin client when --bootstrap-server is given.")
+      .availableIf(bootstrapServerOpt)
+      .withRequiredArg
+      .describedAs("config file")
+      .ofType(classOf[String])
+
+    options = parser.parse(args: _*)
+  }
+
+  /** Abstraction over different ways to perform a leader election */
+  trait Command {
+    /** Elect the preferred leader for the given {@code partitionsForElection}.
+      * If the given {@code partitionsForElection} are None then elect the preferred leader for all partitions.
+      */
+    def electPreferredLeaders(partitionsForElection: Option[Set[TopicPartition]]): Unit
+    def close(): Unit
+  }
+
+  class ZkCommand(zkConnect: String, isSecure: Boolean, timeout: Int)
+    extends Command {
+    var zkClient: KafkaZkClient = null
+
+    val time = Time.SYSTEM
+    zkClient = KafkaZkClient(zkConnect, isSecure, timeout, timeout, Int.MaxValue, time)
+
+    override def electPreferredLeaders(partitionsFromUser: Option[Set[TopicPartition]]): Unit = {
+      try {
+        val topics =
+          partitionsFromUser match {
+            case Some(partitions) =>
+              partitions.map(_.topic).toSet
+            case None =>
+              zkClient.getAllPartitions().map(_.topic)
+          }
+
+        val partitionsFromZk = zkClient.getPartitionsForTopics(topics).flatMap{ case (topic, partitions) =>
+          partitions.map(new TopicPartition(topic, _))
+        }.toSet
+
+        val (validPartitions, invalidPartitions) =
+          partitionsFromUser match {
+            case Some(partitions) =>
+              partitions.partition(partitionsFromZk.contains)
+            case None =>
+              (zkClient.getAllPartitions(), Set.empty)
+          }
+          PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkClient, validPartitions)
+
+        println("Successfully started preferred replica election for partitions %s".format(validPartitions))
+        invalidPartitions.foreach(p => println("Skipping preferred replica leader election for partition %s since it doesn't exist.".format(p)))
+      } catch {
+        case e: Throwable => throw new AdminCommandFailedException("Admin command failed", e)
+      }
+    }
+
+    override def close(): Unit = {
+      if (zkClient != null)
+        zkClient.close()
+    }
+  }
+
+  /** Election via AdminClient.electPreferredLeaders() */
+  class AdminClientCommand(adminClientProps: Properties)
+    extends Command with Logging {
+
+    val adminClient = Admin.create(adminClientProps)
+
+    override def electPreferredLeaders(partitionsFromUser: Option[Set[TopicPartition]]): Unit = {
+      val partitions = partitionsFromUser match {
+        case Some(partitionsFromUser) => partitionsFromUser.asJava
+        case None => null
+      }
+      debug(s"Calling AdminClient.electLeaders(ElectionType.PREFERRED, $partitions)")
+
+      val electionResults = try {
+        adminClient.electLeaders(ElectionType.PREFERRED, partitions).partitions.get.asScala
+      } catch {
+        case e: ExecutionException =>
+          val cause = e.getCause
+          if (cause.isInstanceOf[TimeoutException]) {
+            println("Timeout waiting for election results")
+            throw new AdminCommandFailedException("Timeout waiting for election results", cause)
+          } else if (cause.isInstanceOf[ClusterAuthorizationException]) {
+            println(s"Not authorized to perform leader election")
+            throw new AdminCommandFailedException("Not authorized to perform leader election", cause)
+          }
+
+          throw e
+        case e: Throwable =>
+          // We don't even know the attempted partitions
+          println("Error while making request")
+          e.printStackTrace()
+          return
+      }
+
+      val succeeded = mutable.Set.empty[TopicPartition]
+      val noop = mutable.Set.empty[TopicPartition]
+      val failed = mutable.Map.empty[TopicPartition, Throwable]
+
+      electionResults.foreach[Unit] { case (topicPartition, error) =>
+        if (error.isPresent) {
+          if (error.get.isInstanceOf[ElectionNotNeededException]) {
+            noop += topicPartition
+          } else {
+            failed += topicPartition -> error.get
+          }
+        } else {
+          succeeded += topicPartition
+        }
+      }
+
+      if (!succeeded.isEmpty) {
+        val partitions = succeeded.mkString(", ")
+        println(s"Successfully completed preferred leader election for partitions $partitions")
+      }
+
+      if (!noop.isEmpty) {
+        val partitions = succeeded.mkString(", ")
+        println(s"Preferred replica already elected for partitions $partitions")
+      }
+
+      if (!failed.isEmpty) {
+        val rootException = new AdminCommandFailedException(s"${failed.size} preferred replica(s) could not be elected")
+        failed.foreach { case (topicPartition, exception) =>
+          println(s"Error completing preferred leader election for partition: $topicPartition: $exception")
+          rootException.addSuppressed(exception)
+        }
+        throw rootException
+      }
+    }
+
+    override def close(): Unit = {
+      debug("Closing AdminClient")
+      adminClient.close()
+    }
+  }
+}
+
+class PreferredReplicaLeaderElectionCommand(zkClient: KafkaZkClient, partitionsFromUser: scala.collection.Set[TopicPartition]) {
+  def moveLeaderToPreferredReplica(): Unit = {
+    try {
+      val topics = partitionsFromUser.map(_.topic).toSet
+      val partitionsFromZk = zkClient.getPartitionsForTopics(topics).flatMap { case (topic, partitions) =>
+        partitions.map(new TopicPartition(topic, _))
+      }.toSet
+
+      val (validPartitions, invalidPartitions) = partitionsFromUser.partition(partitionsFromZk.contains)
+      PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkClient, validPartitions)
+
+      println("Successfully started preferred replica election for partitions %s".format(validPartitions))
+      invalidPartitions.foreach(p => println("Skipping preferred replica leader election for partition %s since it doesn't exist.".format(p)))
+    } catch {
+      case e: Throwable => throw new AdminCommandFailedException("Admin command failed", e)
+    }
+  }
+}

core/src/main/scala/kafka/admin/RackAwareMode.scala

@@ -0,0 +1,42 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+package kafka.admin
+
+/**
+  * Mode to control how rack aware replica assignment will be executed
+  */
+object RackAwareMode {
+
+  /**
+    * Ignore all rack information in replica assignment. This is an optional mode used in command line.
+    */
+  case object Disabled extends RackAwareMode
+
+  /**
+    * Assume every broker has rack, or none of the brokers has rack. If only partial brokers have rack, fail fast
+    * in replica assignment. This is the default mode in command line tools (TopicCommand and ReassignPartitionsCommand).
+    */
+  case object Enforced extends RackAwareMode
+
+  /**
+    * Use rack information if every broker has a rack. Otherwise, fallback to Disabled mode. This is used in auto topic
+    * creation.
+    */
+  case object Safe extends RackAwareMode
+}
+
+sealed trait RackAwareMode

core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala

@@ -0,0 +1,684 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.admin
+
+import java.util.Properties
+import java.util.concurrent.ExecutionException
+
+import kafka.common.AdminCommandFailedException
+import kafka.log.LogConfig
+import kafka.log.LogConfig._
+import kafka.server.{ConfigType, DynamicConfig}
+import kafka.utils._
+import kafka.utils.json.JsonValue
+import kafka.zk.{AdminZkClient, KafkaZkClient}
+import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo
+import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterReplicaLogDirsOptions}
+import org.apache.kafka.common.errors.ReplicaNotAvailableException
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.utils.{Time, Utils}
+import org.apache.kafka.common.{TopicPartition, TopicPartitionReplica}
+import org.apache.zookeeper.KeeperException.NodeExistsException
+
+import scala.collection.JavaConverters._
+import scala.collection._
+
+object ReassignPartitionsCommand extends Logging {
+
+  case class Throttle(interBrokerLimit: Long, replicaAlterLogDirsLimit: Long = -1, postUpdateAction: () => Unit = () => ())
+
+  private[admin] val NoThrottle = Throttle(-1, -1)
+  private[admin] val AnyLogDir = "any"
+
+  private[admin] val EarliestVersion = 1
+
+  val helpText = "This tool helps to moves topic partitions between replicas."
+
+  def main(args: Array[String]): Unit = {
+    val opts = validateAndParseArgs(args)
+    val zkConnect = opts.options.valueOf(opts.zkConnectOpt)
+    val time = Time.SYSTEM
+    val zkClient = KafkaZkClient(zkConnect, JaasUtils.isZkSaslEnabled, 30000, 30000, Int.MaxValue, time)
+
+    val adminClientOpt = createAdminClient(opts)
+
+    try {
+      if(opts.options.has(opts.verifyOpt))
+        verifyAssignment(zkClient, adminClientOpt, opts)
+      else if(opts.options.has(opts.generateOpt))
+        generateAssignment(zkClient, opts)
+      else if (opts.options.has(opts.executeOpt))
+        executeAssignment(zkClient, adminClientOpt, opts)
+    } catch {
+      case e: Throwable =>
+        println("Partitions reassignment failed due to " + e.getMessage)
+        println(Utils.stackTrace(e))
+    } finally zkClient.close()
+  }
+
+  private def createAdminClient(opts: ReassignPartitionsCommandOptions): Option[Admin] = {
+    if (opts.options.has(opts.bootstrapServerOpt)) {
+      val props = if (opts.options.has(opts.commandConfigOpt))
+        Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt))
+      else
+        new Properties()
+      props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt))
+      props.putIfAbsent(AdminClientConfig.CLIENT_ID_CONFIG, "reassign-partitions-tool")
+      Some(Admin.create(props))
+    } else {
+      None
+    }
+  }
+
+  def verifyAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], opts: ReassignPartitionsCommandOptions): Unit = {
+    val jsonFile = opts.options.valueOf(opts.reassignmentJsonFileOpt)
+    val jsonString = Utils.readFileAsString(jsonFile)
+    verifyAssignment(zkClient, adminClientOpt, jsonString)
+  }
+
+  def verifyAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], jsonString: String): Unit = {
+    println("Status of partition reassignment: ")
+    val adminZkClient = new AdminZkClient(zkClient)
+    val (partitionsToBeReassigned, replicaAssignment) = parsePartitionReassignmentData(jsonString)
+    val reassignedPartitionsStatus = checkIfPartitionReassignmentSucceeded(zkClient, partitionsToBeReassigned.toMap)
+    val replicasReassignmentStatus = checkIfReplicaReassignmentSucceeded(adminClientOpt, replicaAssignment)
+
+    reassignedPartitionsStatus.foreach { case (topicPartition, status) =>
+      status match {
+        case ReassignmentCompleted =>
+          println("Reassignment of partition %s completed successfully".format(topicPartition))
+        case ReassignmentFailed =>
+          println("Reassignment of partition %s failed".format(topicPartition))
+        case ReassignmentInProgress =>
+          println("Reassignment of partition %s is still in progress".format(topicPartition))
+      }
+    }
+
+    replicasReassignmentStatus.foreach { case (replica, status) =>
+      status match {
+        case ReassignmentCompleted =>
+          println("Reassignment of replica %s completed successfully".format(replica))
+        case ReassignmentFailed =>
+          println("Reassignment of replica %s failed".format(replica))
+        case ReassignmentInProgress =>
+          println("Reassignment of replica %s is still in progress".format(replica))
+      }
+    }
+    removeThrottle(zkClient, reassignedPartitionsStatus, replicasReassignmentStatus, adminZkClient)
+  }
+
+  private[admin] def removeThrottle(zkClient: KafkaZkClient,
+                                    reassignedPartitionsStatus: Map[TopicPartition, ReassignmentStatus],
+                                    replicasReassignmentStatus: Map[TopicPartitionReplica, ReassignmentStatus],
+                                    adminZkClient: AdminZkClient): Unit = {
+
+    //If both partition assignment and replica reassignment have completed remove both the inter-broker and replica-alter-dir throttle
+    if (reassignedPartitionsStatus.forall { case (_, status) => status == ReassignmentCompleted } &&
+        replicasReassignmentStatus.forall { case (_, status) => status == ReassignmentCompleted }) {
+      var changed = false
+
+      //Remove the throttle limit from all brokers in the cluster
+      //(as we no longer know which specific brokers were involved in the move)
+      for (brokerId <- zkClient.getAllBrokersInCluster.map(_.id)) {
+        val configs = adminZkClient.fetchEntityConfig(ConfigType.Broker, brokerId.toString)
+        // bitwise OR as we don't want to short-circuit
+        if (configs.remove(DynamicConfig.Broker.LeaderReplicationThrottledRateProp) != null
+          | configs.remove(DynamicConfig.Broker.FollowerReplicationThrottledRateProp) != null
+          | configs.remove(DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp) != null){
+          adminZkClient.changeBrokerConfig(Seq(brokerId), configs)
+          changed = true
+        }
+      }
+
+      //Remove the list of throttled replicas from all topics with partitions being moved
+      val topics = (reassignedPartitionsStatus.keySet.map(tp => tp.topic) ++ replicasReassignmentStatus.keySet.map(replica => replica.topic)).toSeq.distinct
+      for (topic <- topics) {
+        val configs = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic)
+        // bitwise OR as we don't want to short-circuit
+        if (configs.remove(LogConfig.LeaderReplicationThrottledReplicasProp) != null
+          | configs.remove(LogConfig.FollowerReplicationThrottledReplicasProp) != null) {
+          adminZkClient.changeTopicConfig(topic, configs)
+          changed = true
+        }
+      }
+      if (changed)
+        println("Throttle was removed.")
+    }
+  }
+
+  def generateAssignment(zkClient: KafkaZkClient, opts: ReassignPartitionsCommandOptions): Unit = {
+    val topicsToMoveJsonFile = opts.options.valueOf(opts.topicsToMoveJsonFileOpt)
+    val brokerListToReassign = opts.options.valueOf(opts.brokerListOpt).split(',').map(_.toInt)
+    val duplicateReassignments = CoreUtils.duplicates(brokerListToReassign)
+    if (duplicateReassignments.nonEmpty)
+      throw new AdminCommandFailedException("Broker list contains duplicate entries: %s".format(duplicateReassignments.mkString(",")))
+    val topicsToMoveJsonString = Utils.readFileAsString(topicsToMoveJsonFile)
+    val disableRackAware = opts.options.has(opts.disableRackAware)
+    val (proposedAssignments, currentAssignments) = generateAssignment(zkClient, brokerListToReassign, topicsToMoveJsonString, disableRackAware)
+    println("Current partition replica assignment\n%s\n".format(formatAsReassignmentJson(currentAssignments, Map.empty)))
+    println("Proposed partition reassignment configuration\n%s".format(formatAsReassignmentJson(proposedAssignments, Map.empty)))
+  }
+
+  def generateAssignment(zkClient: KafkaZkClient, brokerListToReassign: Seq[Int], topicsToMoveJsonString: String, disableRackAware: Boolean): (Map[TopicPartition, Seq[Int]], Map[TopicPartition, Seq[Int]]) = {
+    val topicsToReassign = parseTopicsData(topicsToMoveJsonString)
+    val duplicateTopicsToReassign = CoreUtils.duplicates(topicsToReassign)
+    if (duplicateTopicsToReassign.nonEmpty)
+      throw new AdminCommandFailedException("List of topics to reassign contains duplicate entries: %s".format(duplicateTopicsToReassign.mkString(",")))
+    val currentAssignment = zkClient.getReplicaAssignmentForTopics(topicsToReassign.toSet)
+
+    val groupedByTopic = currentAssignment.groupBy { case (tp, _) => tp.topic }
+    val rackAwareMode = if (disableRackAware) RackAwareMode.Disabled else RackAwareMode.Enforced
+    val adminZkClient = new AdminZkClient(zkClient)
+    val brokerMetadatas = adminZkClient.getBrokerMetadatas(rackAwareMode, Some(brokerListToReassign))
+
+    val partitionsToBeReassigned = mutable.Map[TopicPartition, Seq[Int]]()
+    groupedByTopic.foreach { case (topic, assignment) =>
+      val (_, replicas) = assignment.head
+      val assignedReplicas = AdminUtils.assignReplicasToBrokers(brokerMetadatas, assignment.size, replicas.size)
+      partitionsToBeReassigned ++= assignedReplicas.map { case (partition, replicas) =>
+        new TopicPartition(topic, partition) -> replicas
+      }
+    }
+    (partitionsToBeReassigned, currentAssignment)
+  }
+
+  def executeAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], opts: ReassignPartitionsCommandOptions): Unit = {
+    val reassignmentJsonFile =  opts.options.valueOf(opts.reassignmentJsonFileOpt)
+    val reassignmentJsonString = Utils.readFileAsString(reassignmentJsonFile)
+    val interBrokerThrottle = opts.options.valueOf(opts.interBrokerThrottleOpt)
+    val replicaAlterLogDirsThrottle = opts.options.valueOf(opts.replicaAlterLogDirsThrottleOpt)
+    val timeoutMs = opts.options.valueOf(opts.timeoutOpt)
+    executeAssignment(zkClient, adminClientOpt, reassignmentJsonString, Throttle(interBrokerThrottle, replicaAlterLogDirsThrottle), timeoutMs)
+  }
+
+  def executeAssignment(zkClient: KafkaZkClient, adminClientOpt: Option[Admin], reassignmentJsonString: String, throttle: Throttle, timeoutMs: Long = 10000L): Unit = {
+    val (partitionAssignment, replicaAssignment) = parseAndValidate(zkClient, reassignmentJsonString)
+    val adminZkClient = new AdminZkClient(zkClient)
+    val reassignPartitionsCommand = new ReassignPartitionsCommand(zkClient, adminClientOpt, partitionAssignment.toMap, replicaAssignment, adminZkClient)
+
+    // If there is an existing rebalance running, attempt to change its throttle
+    if (zkClient.reassignPartitionsInProgress()) {
+      println("There is an existing assignment running.")
+      reassignPartitionsCommand.maybeLimit(throttle)
+    } else {
+      printCurrentAssignment(zkClient, partitionAssignment.map(_._1.topic))
+      if (throttle.interBrokerLimit >= 0 || throttle.replicaAlterLogDirsLimit >= 0)
+        println(String.format("Warning: You must run Verify periodically, until the reassignment completes, to ensure the throttle is removed. You can also alter the throttle by rerunning the Execute command passing a new value."))
+      if (reassignPartitionsCommand.reassignPartitions(throttle, timeoutMs)) {
+        println("Successfully started reassignment of partitions.")
+      } else
+        println("Failed to reassign partitions %s".format(partitionAssignment))
+    }
+  }
+
+  def printCurrentAssignment(zkClient: KafkaZkClient, topics: Seq[String]): Unit = {
+    // before starting assignment, output the current replica assignment to facilitate rollback
+    val currentPartitionReplicaAssignment = zkClient.getReplicaAssignmentForTopics(topics.toSet)
+    println("Current partition replica assignment\n\n%s\n\nSave this to use as the --reassignment-json-file option during rollback"
+      .format(formatAsReassignmentJson(currentPartitionReplicaAssignment, Map.empty)))
+  }
+
+  def formatAsReassignmentJson(partitionsToBeReassigned: Map[TopicPartition, Seq[Int]],
+                               replicaLogDirAssignment: Map[TopicPartitionReplica, String]): String = {
+    Json.encodeAsString(Map(
+      "version" -> 1,
+      "partitions" -> partitionsToBeReassigned.map { case (tp, replicas) =>
+        Map(
+          "topic" -> tp.topic,
+          "partition" -> tp.partition,
+          "replicas" -> replicas.asJava,
+          "log_dirs" -> replicas.map(r => replicaLogDirAssignment.getOrElse(new TopicPartitionReplica(tp.topic, tp.partition, r), AnyLogDir)).asJava
+        ).asJava
+      }.asJava
+    ).asJava)
+  }
+
+  def parseTopicsData(jsonData: String): Seq[String] = {
+    Json.parseFull(jsonData) match {
+      case Some(js) =>
+        val version = js.asJsonObject.get("version") match {
+          case Some(jsonValue) => jsonValue.to[Int]
+          case None => EarliestVersion
+        }
+        parseTopicsData(version, js)
+      case None => throw new AdminOperationException("The input string is not a valid JSON")
+    }
+  }
+
+  def parseTopicsData(version: Int, js: JsonValue): Seq[String] = {
+    version match {
+      case 1 =>
+        for {
+          partitionsSeq <- js.asJsonObject.get("topics").toSeq
+          p <- partitionsSeq.asJsonArray.iterator
+        } yield p.asJsonObject("topic").to[String]
+      case _ => throw new AdminOperationException(s"Not supported version field value $version")
+    }
+  }
+
+  def parsePartitionReassignmentData(jsonData: String): (Seq[(TopicPartition, Seq[Int])], Map[TopicPartitionReplica, String]) = {
+    Json.parseFull(jsonData) match {
+      case Some(js) =>
+        val version = js.asJsonObject.get("version") match {
+          case Some(jsonValue) => jsonValue.to[Int]
+          case None => EarliestVersion
+        }
+        parsePartitionReassignmentData(version, js)
+      case None => throw new AdminOperationException("The input string is not a valid JSON")
+    }
+  }
+
+  // Parses without deduplicating keys so the data can be checked before allowing reassignment to proceed
+  def parsePartitionReassignmentData(version:Int, jsonData: JsonValue): (Seq[(TopicPartition, Seq[Int])], Map[TopicPartitionReplica, String]) = {
+    version match {
+      case 1 =>
+        val partitionAssignment = mutable.ListBuffer.empty[(TopicPartition, Seq[Int])]
+        val replicaAssignment = mutable.Map.empty[TopicPartitionReplica, String]
+        for {
+          partitionsSeq <- jsonData.asJsonObject.get("partitions").toSeq
+          p <- partitionsSeq.asJsonArray.iterator
+        } {
+          val partitionFields = p.asJsonObject
+          val topic = partitionFields("topic").to[String]
+          val partition = partitionFields("partition").to[Int]
+          val newReplicas = partitionFields("replicas").to[Seq[Int]]
+          val newLogDirs = partitionFields.get("log_dirs") match {
+            case Some(jsonValue) => jsonValue.to[Seq[String]]
+            case None => newReplicas.map(_ => AnyLogDir)
+          }
+          if (newReplicas.size != newLogDirs.size)
+            throw new AdminCommandFailedException(s"Size of replicas list $newReplicas is different from " +
+              s"size of log dirs list $newLogDirs for partition ${new TopicPartition(topic, partition)}")
+          partitionAssignment += (new TopicPartition(topic, partition) -> newReplicas)
+          replicaAssignment ++= newReplicas.zip(newLogDirs).map { case (replica, logDir) =>
+            new TopicPartitionReplica(topic, partition, replica) -> logDir
+          }.filter(_._2 != AnyLogDir)
+        }
+        (partitionAssignment, replicaAssignment)
+      case _ => throw new AdminOperationException(s"Not supported version field value $version")
+    }
+  }
+
+  def parseAndValidate(zkClient: KafkaZkClient, reassignmentJsonString: String): (Seq[(TopicPartition, Seq[Int])], Map[TopicPartitionReplica, String]) = {
+    val (partitionsToBeReassigned, replicaAssignment) = parsePartitionReassignmentData(reassignmentJsonString)
+
+    if (partitionsToBeReassigned.isEmpty)
+      throw new AdminCommandFailedException("Partition reassignment data file is empty")
+    if (partitionsToBeReassigned.exists(_._2.isEmpty)) {
+      throw new AdminCommandFailedException("Partition replica list cannot be empty")
+    }
+    val duplicateReassignedPartitions = CoreUtils.duplicates(partitionsToBeReassigned.map { case (tp, _) => tp })
+    if (duplicateReassignedPartitions.nonEmpty)
+      throw new AdminCommandFailedException("Partition reassignment contains duplicate topic partitions: %s".format(duplicateReassignedPartitions.mkString(",")))
+    val duplicateEntries = partitionsToBeReassigned
+      .map { case (tp, replicas) => (tp, CoreUtils.duplicates(replicas))}
+      .filter { case (_, duplicatedReplicas) => duplicatedReplicas.nonEmpty }
+    if (duplicateEntries.nonEmpty) {
+      val duplicatesMsg = duplicateEntries
+        .map { case (tp, duplicateReplicas) => "%s contains multiple entries for %s".format(tp, duplicateReplicas.mkString(",")) }
+        .mkString(". ")
+      throw new AdminCommandFailedException("Partition replica lists may not contain duplicate entries: %s".format(duplicatesMsg))
+    }
+    // check that all partitions in the proposed assignment exist in the cluster
+    val proposedTopics = partitionsToBeReassigned.map { case (tp, _) => tp.topic }.distinct
+    val existingAssignment = zkClient.getReplicaAssignmentForTopics(proposedTopics.toSet)
+    val nonExistentPartitions = partitionsToBeReassigned.map { case (tp, _) => tp }.filterNot(existingAssignment.contains)
+    if (nonExistentPartitions.nonEmpty)
+      throw new AdminCommandFailedException("The proposed assignment contains non-existent partitions: " +
+        nonExistentPartitions)
+
+    // check that all brokers in the proposed assignment exist in the cluster
+    val existingBrokerIDs = zkClient.getSortedBrokerList
+    val nonExistingBrokerIDs = partitionsToBeReassigned.toMap.values.flatten.filterNot(existingBrokerIDs.contains).toSet
+    if (nonExistingBrokerIDs.nonEmpty)
+      throw new AdminCommandFailedException("The proposed assignment contains non-existent brokerIDs: " + nonExistingBrokerIDs.mkString(","))
+
+    (partitionsToBeReassigned, replicaAssignment)
+  }
+
+  def checkIfPartitionReassignmentSucceeded(zkClient: KafkaZkClient, partitionsToBeReassigned: Map[TopicPartition, Seq[Int]])
+  :Map[TopicPartition, ReassignmentStatus] = {
+    val partitionsBeingReassigned = zkClient.getPartitionReassignment
+    val (beingReassigned, notBeingReassigned) = partitionsToBeReassigned.keys.partition { topicAndPartition =>
+      partitionsBeingReassigned.contains(topicAndPartition)
+    }
+    notBeingReassigned.groupBy(_.topic).flatMap { case (topic, partitions) =>
+      val replicasForTopic = zkClient.getReplicaAssignmentForTopics(immutable.Set(topic))
+      partitions.map { topicAndPartition =>
+        val newReplicas = partitionsToBeReassigned(topicAndPartition)
+        val reassignmentStatus = replicasForTopic.get(topicAndPartition) match {
+          case Some(seq) if seq == newReplicas => ReassignmentCompleted
+          case _ => ReassignmentFailed
+        }
+        (topicAndPartition, reassignmentStatus)
+      }
+    } ++ beingReassigned.map { topicAndPartition =>
+      (topicAndPartition, ReassignmentInProgress)
+    }.toMap
+  }
+
+  private def checkIfReplicaReassignmentSucceeded(adminClientOpt: Option[Admin], replicaAssignment: Map[TopicPartitionReplica, String])
+  :Map[TopicPartitionReplica, ReassignmentStatus] = {
+
+    val replicaLogDirInfos = {
+      if (replicaAssignment.nonEmpty) {
+        val adminClient = adminClientOpt.getOrElse(
+          throw new AdminCommandFailedException("bootstrap-server needs to be provided in order to reassign replica to the specified log directory"))
+        adminClient.describeReplicaLogDirs(replicaAssignment.keySet.asJava).all().get().asScala
+      } else {
+        Map.empty[TopicPartitionReplica, ReplicaLogDirInfo]
+      }
+    }
+
+    replicaAssignment.map { case (replica, newLogDir) =>
+      val status: ReassignmentStatus = replicaLogDirInfos.get(replica) match {
+        case Some(replicaLogDirInfo) =>
+          if (replicaLogDirInfo.getCurrentReplicaLogDir == null) {
+            println(s"Partition ${replica.topic()}-${replica.partition()} is not found in any live log dir on " +
+              s"broker ${replica.brokerId()}. There is likely offline log directory on the broker.")
+            ReassignmentFailed
+          } else if (replicaLogDirInfo.getFutureReplicaLogDir == newLogDir) {
+            ReassignmentInProgress
+          } else if (replicaLogDirInfo.getFutureReplicaLogDir != null) {
+            println(s"Partition ${replica.topic()}-${replica.partition()} on broker ${replica.brokerId()} " +
+              s"is being moved to log dir ${replicaLogDirInfo.getFutureReplicaLogDir} instead of $newLogDir")
+            ReassignmentFailed
+          } else if (replicaLogDirInfo.getCurrentReplicaLogDir == newLogDir) {
+            ReassignmentCompleted
+          } else {
+            println(s"Partition ${replica.topic()}-${replica.partition()} on broker ${replica.brokerId()} " +
+              s"is not being moved from log dir ${replicaLogDirInfo.getCurrentReplicaLogDir} to $newLogDir")
+            ReassignmentFailed
+          }
+        case None =>
+          println(s"Partition ${replica.topic()}-${replica.partition()} is not found in any live log dir on broker ${replica.brokerId()}.")
+          ReassignmentFailed
+      }
+      (replica, status)
+    }
+  }
+
+  def validateAndParseArgs(args: Array[String]): ReassignPartitionsCommandOptions = {
+    val opts = new ReassignPartitionsCommandOptions(args)
+
+    CommandLineUtils.printHelpAndExitIfNeeded(opts, helpText)
+
+    // Should have exactly one action
+    val actions = Seq(opts.generateOpt, opts.executeOpt, opts.verifyOpt).count(opts.options.has _)
+    if(actions != 1)
+      CommandLineUtils.printUsageAndDie(opts.parser, "Command must include exactly one action: --generate, --execute or --verify")
+
+    CommandLineUtils.checkRequiredArgs(opts.parser, opts.options, opts.zkConnectOpt)
+
+    //Validate arguments for each action
+    if(opts.options.has(opts.verifyOpt)) {
+      if(!opts.options.has(opts.reassignmentJsonFileOpt))
+        CommandLineUtils.printUsageAndDie(opts.parser, "If --verify option is used, command must include --reassignment-json-file that was used during the --execute option")
+      CommandLineUtils.checkInvalidArgs(opts.parser, opts.options, opts.verifyOpt, Set(opts.interBrokerThrottleOpt, opts.replicaAlterLogDirsThrottleOpt, opts.topicsToMoveJsonFileOpt, opts.disableRackAware, opts.brokerListOpt))
+    }
+    else if(opts.options.has(opts.generateOpt)) {
+      if(!(opts.options.has(opts.topicsToMoveJsonFileOpt) && opts.options.has(opts.brokerListOpt)))
+        CommandLineUtils.printUsageAndDie(opts.parser, "If --generate option is used, command must include both --topics-to-move-json-file and --broker-list options")
+      CommandLineUtils.checkInvalidArgs(opts.parser, opts.options, opts.generateOpt, Set(opts.interBrokerThrottleOpt, opts.replicaAlterLogDirsThrottleOpt, opts.reassignmentJsonFileOpt))
+    }
+    else if (opts.options.has(opts.executeOpt)){
+      if(!opts.options.has(opts.reassignmentJsonFileOpt))
+        CommandLineUtils.printUsageAndDie(opts.parser, "If --execute option is used, command must include --reassignment-json-file that was output " + "during the --generate option")
+      CommandLineUtils.checkInvalidArgs(opts.parser, opts.options, opts.executeOpt, Set(opts.topicsToMoveJsonFileOpt, opts.disableRackAware, opts.brokerListOpt))
+    }
+    opts
+  }
+
+  class ReassignPartitionsCommandOptions(args: Array[String]) extends CommandDefaultOptions(args)  {
+    val bootstrapServerOpt = parser.accepts("bootstrap-server", "the server(s) to use for bootstrapping. REQUIRED if " +
+                      "an absolute path of the log directory is specified for any replica in the reassignment json file")
+                      .withRequiredArg
+                      .describedAs("Server(s) to use for bootstrapping")
+                      .ofType(classOf[String])
+    val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client.")
+                      .withRequiredArg
+                      .describedAs("Admin client property file")
+                      .ofType(classOf[String])
+    val zkConnectOpt = parser.accepts("zookeeper", "REQUIRED: The connection string for the zookeeper connection in the " +
+                      "form host:port. Multiple URLS can be given to allow fail-over.")
+                      .withRequiredArg
+                      .describedAs("urls")
+                      .ofType(classOf[String])
+    val generateOpt = parser.accepts("generate", "Generate a candidate partition reassignment configuration." +
+                      " Note that this only generates a candidate assignment, it does not execute it.")
+    val executeOpt = parser.accepts("execute", "Kick off the reassignment as specified by the --reassignment-json-file option.")
+    val verifyOpt = parser.accepts("verify", "Verify if the reassignment completed as specified by the --reassignment-json-file option. If there is a throttle engaged for the replicas specified, and the rebalance has completed, the throttle will be removed")
+    val reassignmentJsonFileOpt = parser.accepts("reassignment-json-file", "The JSON file with the partition reassignment configuration" +
+                      "The format to use is - \n" +
+                      "{\"partitions\":\n\t[{\"topic\": \"foo\",\n\t  \"partition\": 1,\n\t  \"replicas\": [1,2,3],\n\t  \"log_dirs\": [\"dir1\",\"dir2\",\"dir3\"] }],\n\"version\":1\n}\n" +
+                      "Note that \"log_dirs\" is optional. When it is specified, its length must equal the length of the replicas list. The value in this list " +
+                      "can be either \"any\" or the absolution path of the log directory on the broker. If absolute log directory path is specified, the replica will be moved to the specified log directory on the broker.")
+                      .withRequiredArg
+                      .describedAs("manual assignment json file path")
+                      .ofType(classOf[String])
+    val topicsToMoveJsonFileOpt = parser.accepts("topics-to-move-json-file", "Generate a reassignment configuration to move the partitions" +
+                      " of the specified topics to the list of brokers specified by the --broker-list option. The format to use is - \n" +
+                      "{\"topics\":\n\t[{\"topic\": \"foo\"},{\"topic\": \"foo1\"}],\n\"version\":1\n}")
+                      .withRequiredArg
+                      .describedAs("topics to reassign json file path")
+                      .ofType(classOf[String])
+    val brokerListOpt = parser.accepts("broker-list", "The list of brokers to which the partitions need to be reassigned" +
+                      " in the form \"0,1,2\". This is required if --topics-to-move-json-file is used to generate reassignment configuration")
+                      .withRequiredArg
+                      .describedAs("brokerlist")
+                      .ofType(classOf[String])
+    val disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment")
+    val interBrokerThrottleOpt = parser.accepts("throttle", "The movement of partitions between brokers will be throttled to this value (bytes/sec). Rerunning with this option, whilst a rebalance is in progress, will alter the throttle value. The throttle rate should be at least 1 KB/s.")
+                      .withRequiredArg()
+                      .describedAs("throttle")
+                      .ofType(classOf[Long])
+                      .defaultsTo(-1)
+    val replicaAlterLogDirsThrottleOpt = parser.accepts("replica-alter-log-dirs-throttle", "The movement of replicas between log directories on the same broker will be throttled to this value (bytes/sec). Rerunning with this option, whilst a rebalance is in progress, will alter the throttle value. The throttle rate should be at least 1 KB/s.")
+                      .withRequiredArg()
+                      .describedAs("replicaAlterLogDirsThrottle")
+                      .ofType(classOf[Long])
+                      .defaultsTo(-1)
+    val timeoutOpt = parser.accepts("timeout", "The maximum time in ms allowed to wait for partition reassignment execution to be successfully initiated")
+                      .withRequiredArg()
+                      .describedAs("timeout")
+                      .ofType(classOf[Long])
+                      .defaultsTo(10000)
+    options = parser.parse(args : _*)
+  }
+}
+
+class ReassignPartitionsCommand(zkClient: KafkaZkClient,
+                                adminClientOpt: Option[Admin],
+                                proposedPartitionAssignment: Map[TopicPartition, Seq[Int]],
+                                proposedReplicaAssignment: Map[TopicPartitionReplica, String] = Map.empty,
+                                adminZkClient: AdminZkClient)
+  extends Logging {
+
+  import ReassignPartitionsCommand._
+
+  def existingAssignment(): Map[TopicPartition, Seq[Int]] = {
+    val proposedTopics = proposedPartitionAssignment.keySet.map(_.topic).toSeq
+    zkClient.getReplicaAssignmentForTopics(proposedTopics.toSet)
+  }
+
+  private def maybeThrottle(throttle: Throttle): Unit = {
+    if (throttle.interBrokerLimit >= 0)
+      assignThrottledReplicas(existingAssignment(), proposedPartitionAssignment, adminZkClient)
+    maybeLimit(throttle)
+    if (throttle.interBrokerLimit >= 0 || throttle.replicaAlterLogDirsLimit >= 0)
+      throttle.postUpdateAction()
+    if (throttle.interBrokerLimit >= 0)
+      println(s"The inter-broker throttle limit was set to ${throttle.interBrokerLimit} B/s")
+    if (throttle.replicaAlterLogDirsLimit >= 0)
+      println(s"The replica-alter-dir throttle limit was set to ${throttle.replicaAlterLogDirsLimit} B/s")
+  }
+
+  /**
+    * Limit the throttle on currently moving replicas. Note that this command can use used to alter the throttle, but
+    * it may not alter all limits originally set, if some of the brokers have completed their rebalance.
+    */
+  def maybeLimit(throttle: Throttle): Unit = {
+    if (throttle.interBrokerLimit >= 0 || throttle.replicaAlterLogDirsLimit >= 0) {
+      val existingBrokers = existingAssignment().values.flatten.toSeq
+      val proposedBrokers = proposedPartitionAssignment.values.flatten.toSeq ++ proposedReplicaAssignment.keys.toSeq.map(_.brokerId())
+      val brokers = (existingBrokers ++ proposedBrokers).distinct
+
+      for (id <- brokers) {
+        val configs = adminZkClient.fetchEntityConfig(ConfigType.Broker, id.toString)
+        if (throttle.interBrokerLimit >= 0) {
+          configs.put(DynamicConfig.Broker.LeaderReplicationThrottledRateProp, throttle.interBrokerLimit.toString)
+          configs.put(DynamicConfig.Broker.FollowerReplicationThrottledRateProp, throttle.interBrokerLimit.toString)
+        }
+        if (throttle.replicaAlterLogDirsLimit >= 0)
+          configs.put(DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp, throttle.replicaAlterLogDirsLimit.toString)
+
+        adminZkClient.changeBrokerConfig(Seq(id), configs)
+      }
+    }
+  }
+
+  /** Set throttles to replicas that are moving. Note: this method should only be used when the assignment is initiated. */
+  private[admin] def assignThrottledReplicas(existingPartitionAssignment: Map[TopicPartition, Seq[Int]],
+                                             proposedPartitionAssignment: Map[TopicPartition, Seq[Int]],
+                                             adminZkClient: AdminZkClient): Unit = {
+    for (topic <- proposedPartitionAssignment.keySet.map(_.topic).toSeq.distinct) {
+      val existingPartitionAssignmentForTopic = existingPartitionAssignment.filter { case (tp, _) => tp.topic == topic }
+      val proposedPartitionAssignmentForTopic = proposedPartitionAssignment.filter { case (tp, _) => tp.topic == topic }
+
+      //Apply leader throttle to all replicas that exist before the re-balance.
+      val leader = format(preRebalanceReplicaForMovingPartitions(existingPartitionAssignmentForTopic, proposedPartitionAssignmentForTopic))
+
+      //Apply follower throttle to all "move destinations".
+      val follower = format(postRebalanceReplicasThatMoved(existingPartitionAssignmentForTopic, proposedPartitionAssignmentForTopic))
+
+      val configs = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic)
+      configs.put(LeaderReplicationThrottledReplicasProp, leader)
+      configs.put(FollowerReplicationThrottledReplicasProp, follower)
+      adminZkClient.changeTopicConfig(topic, configs)
+
+      debug(s"Updated leader-throttled replicas for topic $topic with: $leader")
+      debug(s"Updated follower-throttled replicas for topic $topic with: $follower")
+    }
+  }
+
+  private def postRebalanceReplicasThatMoved(existing: Map[TopicPartition, Seq[Int]], proposed: Map[TopicPartition, Seq[Int]]): Map[TopicPartition, Seq[Int]] = {
+    //For each partition in the proposed list, filter out any replicas that exist now, and hence aren't being moved.
+    proposed.map { case (tp, proposedReplicas) =>
+      tp -> (proposedReplicas.toSet -- existing(tp)).toSeq
+    }
+  }
+
+  private def preRebalanceReplicaForMovingPartitions(existing: Map[TopicPartition, Seq[Int]], proposed: Map[TopicPartition, Seq[Int]]): Map[TopicPartition, Seq[Int]] = {
+    def moving(before: Seq[Int], after: Seq[Int]) = (after.toSet -- before.toSet).nonEmpty
+    //For any moving partition, throttle all the original (pre move) replicas (as any one might be a leader)
+    existing.filter { case (tp, preMoveReplicas) =>
+      proposed.contains(tp) && moving(preMoveReplicas, proposed(tp))
+    }
+  }
+
+  def format(moves: Map[TopicPartition, Seq[Int]]): String =
+    moves.flatMap { case (tp, moves) =>
+      moves.map(replicaId => s"${tp.partition}:${replicaId}")
+    }.mkString(",")
+
+  private def alterReplicaLogDirsIgnoreReplicaNotAvailable(replicaAssignment: Map[TopicPartitionReplica, String],
+                                                           adminClient: Admin,
+                                                           timeoutMs: Long): Set[TopicPartitionReplica] = {
+    val alterReplicaLogDirsResult = adminClient.alterReplicaLogDirs(replicaAssignment.asJava, new AlterReplicaLogDirsOptions().timeoutMs(timeoutMs.toInt))
+    val replicasAssignedToFutureDir = alterReplicaLogDirsResult.values().asScala.flatMap { case (replica, future) => {
+      try {
+        future.get()
+        Some(replica)
+      } catch {
+        case t: ExecutionException =>
+          t.getCause match {
+            case _: ReplicaNotAvailableException => None // It is OK if the replica is not available at this moment
+            case e: Throwable => throw new AdminCommandFailedException(s"Failed to alter dir for $replica", e)
+          }
+      }
+    }}
+    replicasAssignedToFutureDir.toSet
+  }
+
+  def reassignPartitions(throttle: Throttle = NoThrottle, timeoutMs: Long = 10000L): Boolean = {
+    maybeThrottle(throttle)
+    try {
+      val validPartitions = proposedPartitionAssignment.groupBy(_._1.topic())
+        .flatMap { case (topic, topicPartitionReplicas) =>
+          validatePartition(zkClient, topic, topicPartitionReplicas)
+        }
+      if (validPartitions.isEmpty) false
+      else {
+        if (proposedReplicaAssignment.nonEmpty && adminClientOpt.isEmpty)
+          throw new AdminCommandFailedException("bootstrap-server needs to be provided in order to reassign replica to the specified log directory")
+        val startTimeMs = System.currentTimeMillis()
+
+        // Send AlterReplicaLogDirsRequest to allow broker to create replica in the right log dir later if the replica has not been created yet.
+        if (proposedReplicaAssignment.nonEmpty)
+          alterReplicaLogDirsIgnoreReplicaNotAvailable(proposedReplicaAssignment, adminClientOpt.get, timeoutMs)
+
+        // Create reassignment znode so that controller will send LeaderAndIsrRequest to create replica in the broker
+        zkClient.createPartitionReassignment(validPartitions.map({case (key, value) => (new TopicPartition(key.topic, key.partition), value)}).toMap)
+
+        // Send AlterReplicaLogDirsRequest again to make sure broker will start to move replica to the specified log directory.
+        // It may take some time for controller to create replica in the broker. Retry if the replica has not been created.
+        var remainingTimeMs = startTimeMs + timeoutMs - System.currentTimeMillis()
+        val replicasAssignedToFutureDir = mutable.Set.empty[TopicPartitionReplica]
+        while (remainingTimeMs > 0 && replicasAssignedToFutureDir.size < proposedReplicaAssignment.size) {
+          replicasAssignedToFutureDir ++= alterReplicaLogDirsIgnoreReplicaNotAvailable(
+            proposedReplicaAssignment.filter { case (replica, _) => !replicasAssignedToFutureDir.contains(replica) },
+            adminClientOpt.get, remainingTimeMs)
+          Thread.sleep(100)
+          remainingTimeMs = startTimeMs + timeoutMs - System.currentTimeMillis()
+        }
+        replicasAssignedToFutureDir.size == proposedReplicaAssignment.size
+      }
+    } catch {
+      case _: NodeExistsException =>
+        val partitionsBeingReassigned = zkClient.getPartitionReassignment()
+        throw new AdminCommandFailedException("Partition reassignment currently in " +
+          "progress for %s. Aborting operation".format(partitionsBeingReassigned))
+    }
+  }
+
+  def validatePartition(zkClient: KafkaZkClient, topic: String, topicPartitionReplicas: Map[TopicPartition, Seq[Int]])
+  :Map[TopicPartition, Seq[Int]] = {
+    // check if partition exists
+    val partitionsOpt = zkClient.getPartitionsForTopics(immutable.Set(topic)).get(topic)
+    topicPartitionReplicas.filter { case (topicPartition, _) =>
+      partitionsOpt match {
+        case Some(partitions) =>
+          if (partitions.contains(topicPartition.partition())) {
+            true
+          } else {
+            error("Skipping reassignment of partition [%s,%d] ".format(topic, topicPartition.partition()) +
+              "since it doesn't exist")
+            false
+          }
+        case None => error("Skipping reassignment of partition " +
+          "[%s,%d] since topic %s doesn't exist".format(topic, topicPartition.partition(), topic))
+          false
+      }
+    }
+  }
+}
+
+sealed trait ReassignmentStatus { def status: Int }
+case object ReassignmentCompleted extends ReassignmentStatus { val status = 1 }
+case object ReassignmentInProgress extends ReassignmentStatus { val status = 0 }
+case object ReassignmentFailed extends ReassignmentStatus { val status = -1 }
+

core/src/main/scala/kafka/admin/TopicCommand.scala

@@ -0,0 +1,750 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import java.util
+import java.util.{Collections, Properties}
+
+import joptsimple._
+import kafka.common.AdminCommandFailedException
+import kafka.log.LogConfig
+import kafka.server.ConfigType
+import kafka.utils.Implicits._
+import kafka.utils._
+import kafka.zk.{AdminZkClient, KafkaZkClient}
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.clients.admin.{Admin, ConfigEntry, ListTopicsOptions, NewPartitions, NewTopic, PartitionReassignment, Config => JConfig}
+import org.apache.kafka.common.{Node, TopicPartition, TopicPartitionInfo}
+import org.apache.kafka.common.config.ConfigResource.Type
+import org.apache.kafka.common.config.{ConfigResource, TopicConfig}
+import org.apache.kafka.common.errors.{ClusterAuthorizationException, InvalidTopicException, TopicExistsException, UnsupportedVersionException}
+import org.apache.kafka.common.internals.Topic
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.utils.{Time, Utils}
+import org.apache.zookeeper.KeeperException.NodeExistsException
+
+import scala.collection.JavaConverters._
+import scala.collection._
+import scala.compat.java8.OptionConverters._
+import scala.concurrent.ExecutionException
+import scala.io.StdIn
+
+object TopicCommand extends Logging {
+
+  def main(args: Array[String]): Unit = {
+
+    val opts = new TopicCommandOptions(args)
+    opts.checkArgs()
+
+    val topicService = if (opts.zkConnect.isDefined)
+      ZookeeperTopicService(opts.zkConnect)
+    else
+      AdminClientTopicService(opts.commandConfig, opts.bootstrapServer)
+
+    var exitCode = 0
+    try {
+      if (opts.hasCreateOption)
+        topicService.createTopic(opts)
+      else if (opts.hasAlterOption)
+        topicService.alterTopic(opts)
+      else if (opts.hasListOption)
+        topicService.listTopics(opts)
+      else if (opts.hasDescribeOption)
+        topicService.describeTopic(opts)
+      else if (opts.hasDeleteOption)
+        topicService.deleteTopic(opts)
+    } catch {
+      case e: Throwable =>
+        println("Error while executing topic command : " + e.getMessage)
+        error(Utils.stackTrace(e))
+        exitCode = 1
+    } finally {
+      topicService.close()
+      Exit.exit(exitCode)
+    }
+  }
+
+  class CommandTopicPartition(opts: TopicCommandOptions) {
+    val name: String = opts.topic.get
+    val partitions: Option[Integer] = opts.partitions
+    val replicationFactor: Option[Integer] = opts.replicationFactor
+    val replicaAssignment: Option[Map[Int, List[Int]]] = opts.replicaAssignment
+    val configsToAdd: Properties = parseTopicConfigsToBeAdded(opts)
+    val configsToDelete: Seq[String] = parseTopicConfigsToBeDeleted(opts)
+    val rackAwareMode: RackAwareMode = opts.rackAwareMode
+
+    def hasReplicaAssignment: Boolean = replicaAssignment.isDefined
+    def hasPartitions: Boolean = partitions.isDefined
+    def ifTopicDoesntExist(): Boolean = opts.ifNotExists
+  }
+
+  case class TopicDescription(topic: String,
+                              numPartitions: Int,
+                              replicationFactor: Int,
+                              config: JConfig,
+                              markedForDeletion: Boolean) {
+
+    def printDescription(): Unit = {
+      val configsAsString = config.entries.asScala.filter(!_.isDefault).map { ce => s"${ce.name}=${ce.value}" }.mkString(",")
+      print(s"Topic: $topic")
+      print(s"\tPartitionCount: $numPartitions")
+      print(s"\tReplicationFactor: $replicationFactor")
+      print(s"\tConfigs: $configsAsString")
+      print(if (markedForDeletion) "\tMarkedForDeletion: true" else "")
+      println()
+    }
+  }
+
+  case class PartitionDescription(topic: String,
+                                  info: TopicPartitionInfo,
+                                  config: Option[JConfig],
+                                  markedForDeletion: Boolean,
+                                  reassignment: Option[PartitionReassignment]) {
+
+    private def minIsrCount: Option[Int] = {
+      config.map(_.get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG).value.toInt)
+    }
+
+    def isUnderReplicated: Boolean = {
+      getReplicationFactor(info, reassignment) - info.isr.size > 0
+    }
+
+    private def hasLeader: Boolean = {
+      info.leader != null
+    }
+
+    def isUnderMinIsr: Boolean = {
+      !hasLeader || minIsrCount.exists(info.isr.size < _)
+    }
+
+    def isAtMinIsrPartitions: Boolean =  {
+      minIsrCount.contains(info.isr.size)
+    }
+
+    def hasUnavailablePartitions(liveBrokers: Set[Int]): Boolean = {
+      !hasLeader || !liveBrokers.contains(info.leader.id)
+    }
+
+    def printDescription(): Unit = {
+      print("\tTopic: " + topic)
+      print("\tPartition: " + info.partition)
+      print("\tLeader: " + (if (hasLeader) info.leader.id else "none"))
+      print("\tReplicas: " + info.replicas.asScala.map(_.id).mkString(","))
+      print("\tIsr: " + info.isr.asScala.map(_.id).mkString(","))
+      print(if (markedForDeletion) "\tMarkedForDeletion: true" else "")
+      println()
+    }
+
+  }
+
+  class DescribeOptions(opts: TopicCommandOptions, liveBrokers: Set[Int]) {
+    val describeConfigs: Boolean =
+      !opts.reportUnavailablePartitions &&
+      !opts.reportUnderReplicatedPartitions &&
+      !opts.reportUnderMinIsrPartitions &&
+      !opts.reportAtMinIsrPartitions
+    val describePartitions: Boolean = !opts.reportOverriddenConfigs
+
+    private def shouldPrintUnderReplicatedPartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportUnderReplicatedPartitions && partitionDescription.isUnderReplicated
+    }
+    private def shouldPrintUnavailablePartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportUnavailablePartitions && partitionDescription.hasUnavailablePartitions(liveBrokers)
+    }
+    private def shouldPrintUnderMinIsrPartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportUnderMinIsrPartitions && partitionDescription.isUnderMinIsr
+    }
+    private def shouldPrintAtMinIsrPartitions(partitionDescription: PartitionDescription): Boolean = {
+      opts.reportAtMinIsrPartitions && partitionDescription.isAtMinIsrPartitions
+    }
+
+    private def shouldPrintTopicPartition(partitionDesc: PartitionDescription): Boolean = {
+      describeConfigs ||
+        shouldPrintUnderReplicatedPartitions(partitionDesc) ||
+        shouldPrintUnavailablePartitions(partitionDesc) ||
+        shouldPrintUnderMinIsrPartitions(partitionDesc) ||
+        shouldPrintAtMinIsrPartitions(partitionDesc)
+    }
+
+    def maybePrintPartitionDescription(desc: PartitionDescription): Unit = {
+      if (shouldPrintTopicPartition(desc))
+        desc.printDescription()
+    }
+  }
+
+  trait TopicService extends AutoCloseable {
+    def createTopic(opts: TopicCommandOptions): Unit = {
+      val topic = new CommandTopicPartition(opts)
+      if (Topic.hasCollisionChars(topic.name))
+        println("WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could " +
+          "collide. To avoid issues it is best to use either, but not both.")
+      createTopic(topic)
+    }
+    def createTopic(topic: CommandTopicPartition): Unit
+    def listTopics(opts: TopicCommandOptions): Unit
+    def alterTopic(opts: TopicCommandOptions): Unit
+    def describeTopic(opts: TopicCommandOptions): Unit
+    def deleteTopic(opts: TopicCommandOptions): Unit
+    def getTopics(topicWhitelist: Option[String], excludeInternalTopics: Boolean = false): Seq[String]
+  }
+
+  object AdminClientTopicService {
+    def createAdminClient(commandConfig: Properties, bootstrapServer: Option[String]): Admin = {
+      bootstrapServer match {
+        case Some(serverList) => commandConfig.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, serverList)
+        case None =>
+      }
+      Admin.create(commandConfig)
+    }
+
+    def apply(commandConfig: Properties, bootstrapServer: Option[String]): AdminClientTopicService =
+      new AdminClientTopicService(createAdminClient(commandConfig, bootstrapServer))
+  }
+
+  case class AdminClientTopicService private (adminClient: Admin) extends TopicService {
+
+    override def createTopic(topic: CommandTopicPartition): Unit = {
+      if (topic.replicationFactor.exists(rf => rf > Short.MaxValue || rf < 1))
+        throw new IllegalArgumentException(s"The replication factor must be between 1 and ${Short.MaxValue} inclusive")
+      if (topic.partitions.exists(partitions => partitions < 1))
+        throw new IllegalArgumentException(s"The partitions must be greater than 0")
+
+      if (!adminClient.listTopics().names().get().contains(topic.name)) {
+        val newTopic = if (topic.hasReplicaAssignment)
+          new NewTopic(topic.name, asJavaReplicaReassignment(topic.replicaAssignment.get))
+        else {
+          new NewTopic(
+            topic.name,
+            topic.partitions.asJava,
+            topic.replicationFactor.map(_.toShort).map(Short.box).asJava)
+        }
+
+        val configsMap = topic.configsToAdd.stringPropertyNames()
+          .asScala
+          .map(name => name -> topic.configsToAdd.getProperty(name))
+          .toMap.asJava
+
+        newTopic.configs(configsMap)
+        val createResult = adminClient.createTopics(Collections.singleton(newTopic))
+        createResult.all().get()
+        println(s"Created topic ${topic.name}.")
+      } else {
+        throw new IllegalArgumentException(s"Topic ${topic.name} already exists")
+      }
+    }
+
+    override def listTopics(opts: TopicCommandOptions): Unit = {
+      println(getTopics(opts.topic, opts.excludeInternalTopics).mkString("\n"))
+    }
+
+    override def alterTopic(opts: TopicCommandOptions): Unit = {
+      val topic = new CommandTopicPartition(opts)
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      ensureTopicExists(topics, opts.topic)
+      val topicsInfo = adminClient.describeTopics(topics.asJavaCollection).values()
+      adminClient.createPartitions(topics.map {topicName =>
+        if (topic.hasReplicaAssignment) {
+          val startPartitionId = topicsInfo.get(topicName).get().partitions().size()
+          val newAssignment = {
+            val replicaMap = topic.replicaAssignment.get.drop(startPartitionId)
+            new util.ArrayList(replicaMap.map(p => p._2.asJava).asJavaCollection).asInstanceOf[util.List[util.List[Integer]]]
+          }
+          topicName -> NewPartitions.increaseTo(topic.partitions.get, newAssignment)
+        } else {
+          topicName -> NewPartitions.increaseTo(topic.partitions.get)
+        }}.toMap.asJava).all().get()
+    }
+
+    private def listAllReassignments(): Map[TopicPartition, PartitionReassignment] = {
+      try {
+        adminClient.listPartitionReassignments().reassignments().get().asScala
+      } catch {
+        case e: ExecutionException =>
+          e.getCause match {
+            case ex @ (_: UnsupportedVersionException | _: ClusterAuthorizationException) =>
+              logger.debug(s"Couldn't query reassignments through the AdminClient API: ${ex.getMessage}", ex)
+              Map()
+            case t => throw t
+          }
+      }
+    }
+
+    override def describeTopic(opts: TopicCommandOptions): Unit = {
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      val allConfigs = adminClient.describeConfigs(topics.map(new ConfigResource(Type.TOPIC, _)).asJavaCollection).values()
+      val liveBrokers = adminClient.describeCluster().nodes().get().asScala.map(_.id())
+      val reassignments = listAllReassignments()
+      val topicDescriptions = adminClient.describeTopics(topics.asJavaCollection).all().get().values().asScala
+      val describeOptions = new DescribeOptions(opts, liveBrokers.toSet)
+
+      for (td <- topicDescriptions) {
+        val topicName = td.name
+        val config = allConfigs.get(new ConfigResource(Type.TOPIC, topicName)).get()
+        val sortedPartitions = td.partitions.asScala.sortBy(_.partition)
+
+        if (describeOptions.describeConfigs) {
+          val hasNonDefault = config.entries().asScala.exists(!_.isDefault)
+          if (!opts.reportOverriddenConfigs || hasNonDefault) {
+            val numPartitions = td.partitions().size
+            val firstPartition = td.partitions.iterator.next()
+            val reassignment = reassignments.get(new TopicPartition(td.name, firstPartition.partition))
+            val topicDesc = TopicDescription(topicName, numPartitions, getReplicationFactor(firstPartition, reassignment), config, markedForDeletion = false)
+            topicDesc.printDescription()
+          }
+        }
+
+        if (describeOptions.describePartitions) {
+          for (partition <- sortedPartitions) {
+            val reassignment = reassignments.get(new TopicPartition(td.name, partition.partition))
+            val partitionDesc = PartitionDescription(topicName, partition, Some(config), markedForDeletion = false, reassignment)
+            describeOptions.maybePrintPartitionDescription(partitionDesc)
+          }
+        }
+      }
+    }
+
+    override def deleteTopic(opts: TopicCommandOptions): Unit = {
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      ensureTopicExists(topics, opts.topic)
+      adminClient.deleteTopics(topics.asJavaCollection).all().get()
+    }
+
+    override def getTopics(topicWhitelist: Option[String], excludeInternalTopics: Boolean = false): Seq[String] = {
+      val allTopics = if (excludeInternalTopics) {
+        adminClient.listTopics()
+      } else {
+        adminClient.listTopics(new ListTopicsOptions().listInternal(true))
+      }
+      doGetTopics(allTopics.names().get().asScala.toSeq.sorted, topicWhitelist, excludeInternalTopics)
+    }
+
+    override def close(): Unit = adminClient.close()
+  }
+
+  object ZookeeperTopicService {
+    def apply(zkConnect: Option[String]): ZookeeperTopicService =
+      new ZookeeperTopicService(KafkaZkClient(zkConnect.get, JaasUtils.isZkSaslEnabled, 30000, 30000,
+        Int.MaxValue, Time.SYSTEM))
+  }
+
+  case class ZookeeperTopicService(zkClient: KafkaZkClient) extends TopicService {
+
+    override def createTopic(topic: CommandTopicPartition): Unit = {
+      val adminZkClient = new AdminZkClient(zkClient)
+      try {
+        if (topic.hasReplicaAssignment)
+          adminZkClient.createTopicWithAssignment(topic.name, topic.configsToAdd, topic.replicaAssignment.get)
+        else
+          adminZkClient.createTopic(topic.name, topic.partitions.get, topic.replicationFactor.get, topic.configsToAdd, topic.rackAwareMode)
+        println(s"Created topic ${topic.name}.")
+      } catch  {
+        case e: TopicExistsException => if (!topic.ifTopicDoesntExist()) throw e
+      }
+    }
+
+    override def listTopics(opts: TopicCommandOptions): Unit = {
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      for(topic <- topics) {
+        if (zkClient.isTopicMarkedForDeletion(topic))
+          println(s"$topic - marked for deletion")
+        else
+          println(topic)
+      }
+    }
+
+    override def alterTopic(opts: TopicCommandOptions): Unit = {
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      val tp = new CommandTopicPartition(opts)
+      ensureTopicExists(topics, opts.topic, !opts.ifExists)
+      val adminZkClient = new AdminZkClient(zkClient)
+      topics.foreach { topic =>
+        val configs = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic)
+        if(opts.topicConfig.isDefined || opts.configsToDelete.isDefined) {
+          println("WARNING: Altering topic configuration from this script has been deprecated and may be removed in future releases.")
+          println("         Going forward, please use kafka-configs.sh for this functionality")
+
+          // compile the final set of configs
+          configs ++= tp.configsToAdd
+          tp.configsToDelete.foreach(config => configs.remove(config))
+          adminZkClient.changeTopicConfig(topic, configs)
+          println(s"Updated config for topic $topic.")
+        }
+
+        if(tp.hasPartitions) {
+          if (Topic.isInternal(topic)) {
+            throw new IllegalArgumentException(s"The number of partitions for the internal topic $topic cannot be changed.")
+          }
+          println("WARNING: If partitions are increased for a topic that has a key, the partition " +
+            "logic or ordering of the messages will be affected")
+          val existingAssignment = zkClient.getFullReplicaAssignmentForTopics(immutable.Set(topic)).map {
+            case (topicPartition, assignment) => topicPartition.partition -> assignment
+          }
+          if (existingAssignment.isEmpty)
+            throw new InvalidTopicException(s"The topic $topic does not exist")
+          val newAssignment = tp.replicaAssignment.getOrElse(Map()).drop(existingAssignment.size)
+          val allBrokers = adminZkClient.getBrokerMetadatas()
+          val partitions: Integer = tp.partitions.getOrElse(1)
+          adminZkClient.addPartitions(topic, existingAssignment, allBrokers, partitions, Option(newAssignment).filter(_.nonEmpty))
+          println("Adding partitions succeeded!")
+        }
+      }
+    }
+
+    override def describeTopic(opts: TopicCommandOptions): Unit = {
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      ensureTopicExists(topics, opts.topic, !opts.ifExists)
+      val liveBrokers = zkClient.getAllBrokersInCluster.map(broker => broker.id -> broker).toMap
+      val liveBrokerIds = liveBrokers.keySet
+      val describeOptions = new DescribeOptions(opts, liveBrokerIds)
+      val adminZkClient = new AdminZkClient(zkClient)
+
+      for (topic <- topics) {
+        zkClient.getPartitionAssignmentForTopics(immutable.Set(topic)).get(topic) match {
+          case Some(topicPartitionAssignment) =>
+            val markedForDeletion = zkClient.isTopicMarkedForDeletion(topic)
+            if (describeOptions.describeConfigs) {
+              val configs = adminZkClient.fetchEntityConfig(ConfigType.Topic, topic).asScala
+              if (!opts.reportOverriddenConfigs || configs.nonEmpty) {
+                val numPartitions = topicPartitionAssignment.size
+                val replicationFactor = topicPartitionAssignment.head._2.replicas.size
+                val config = new JConfig(configs.map{ case (k, v) => new ConfigEntry(k, v) }.asJavaCollection)
+                val topicDesc = TopicDescription(topic, numPartitions, replicationFactor, config, markedForDeletion)
+                topicDesc.printDescription()
+              }
+            }
+            if (describeOptions.describePartitions) {
+              for ((partitionId, replicaAssignment) <- topicPartitionAssignment.toSeq.sortBy(_._1)) {
+                val assignedReplicas = replicaAssignment.replicas
+                val tp = new TopicPartition(topic, partitionId)
+                val (leaderOpt, isr) =  zkClient.getTopicPartitionState(tp).map(_.leaderAndIsr) match {
+                  case Some(leaderAndIsr) => (leaderAndIsr.leaderOpt, leaderAndIsr.isr)
+                  case None => (None, Seq.empty[Int])
+                }
+
+                def asNode(brokerId: Int): Node = {
+                  liveBrokers.get(brokerId) match {
+                    case Some(broker) => broker.node(broker.endPoints.head.listenerName)
+                    case None => new Node(brokerId, "", -1)
+                  }
+                }
+
+                val info = new TopicPartitionInfo(partitionId, leaderOpt.map(asNode).orNull,
+                  assignedReplicas.map(asNode).toList.asJava,
+                  isr.map(asNode).toList.asJava)
+
+                val partitionDesc = PartitionDescription(topic, info, config = None, markedForDeletion, reassignment = None)
+                describeOptions.maybePrintPartitionDescription(partitionDesc)
+              }
+            }
+          case None =>
+            println("Topic " + topic + " doesn't exist!")
+        }
+      }
+    }
+
+    override def deleteTopic(opts: TopicCommandOptions): Unit = {
+      val topics = getTopics(opts.topic, opts.excludeInternalTopics)
+      ensureTopicExists(topics, opts.topic, !opts.ifExists)
+      topics.foreach { topic =>
+        try {
+          if (Topic.isInternal(topic)) {
+            throw new AdminOperationException(s"Topic $topic is a kafka internal topic and is not allowed to be marked for deletion.")
+          } else {
+            zkClient.createDeleteTopicPath(topic)
+            println(s"Topic $topic is marked for deletion.")
+            println("Note: This will have no impact if delete.topic.enable is not set to true.")
+          }
+        } catch {
+          case _: NodeExistsException =>
+            println(s"Topic $topic is already marked for deletion.")
+          case e: AdminOperationException =>
+            throw e
+          case e: Throwable =>
+            throw new AdminOperationException(s"Error while deleting topic $topic", e)
+        }
+      }
+    }
+
+    override def getTopics(topicWhitelist: Option[String], excludeInternalTopics: Boolean = false): Seq[String] = {
+      val allTopics = zkClient.getAllTopicsInCluster.toSeq.sorted
+      doGetTopics(allTopics, topicWhitelist, excludeInternalTopics)
+    }
+
+    override def close(): Unit = zkClient.close()
+  }
+
+  /**
+    * ensures topic existence and throws exception if topic doesn't exist
+    *
+    * @param foundTopics Topics that were found to match the requested topic name.
+    * @param requestedTopic Name of the topic that was requested.
+    * @param requireTopicExists Indicates if the topic needs to exist for the operation to be successful.
+    *                           If set to true, the command will throw an exception if the topic with the
+    *                           requested name does not exist.
+    */
+  private def ensureTopicExists(foundTopics: Seq[String], requestedTopic: Option[String], requireTopicExists: Boolean = true): Unit = {
+    // If no topic name was mentioned, do not need to throw exception.
+    if (requestedTopic.isDefined && requireTopicExists && foundTopics.isEmpty) {
+      // If given topic doesn't exist then throw exception
+      throw new IllegalArgumentException(s"Topic '${requestedTopic.get}' does not exist as expected")
+    }
+  }
+
+  private def doGetTopics(allTopics: Seq[String], topicWhitelist: Option[String], excludeInternalTopics: Boolean): Seq[String] = {
+    if (topicWhitelist.isDefined) {
+      val topicsFilter = Whitelist(topicWhitelist.get)
+      allTopics.filter(topicsFilter.isTopicAllowed(_, excludeInternalTopics))
+    } else
+    allTopics.filterNot(Topic.isInternal(_) && excludeInternalTopics)
+  }
+
+
+  def parseTopicConfigsToBeAdded(opts: TopicCommandOptions): Properties = {
+    val configsToBeAdded = opts.topicConfig.getOrElse(Collections.emptyList()).asScala.map(_.split("""\s*=\s*"""))
+    require(configsToBeAdded.forall(config => config.length == 2),
+      "Invalid topic config: all configs to be added must be in the format \"key=val\".")
+    val props = new Properties
+    configsToBeAdded.foreach(pair => props.setProperty(pair(0).trim, pair(1).trim))
+    LogConfig.validate(props)
+    if (props.containsKey(LogConfig.MessageFormatVersionProp)) {
+      println(s"WARNING: The configuration ${LogConfig.MessageFormatVersionProp}=${props.getProperty(LogConfig.MessageFormatVersionProp)} is specified. " +
+        s"This configuration will be ignored if the version is newer than the inter.broker.protocol.version specified in the broker.")
+    }
+    props
+  }
+
+  def parseTopicConfigsToBeDeleted(opts: TopicCommandOptions): Seq[String] = {
+    val configsToBeDeleted = opts.configsToDelete.getOrElse(Collections.emptyList()).asScala.map(_.trim())
+    val propsToBeDeleted = new Properties
+    configsToBeDeleted.foreach(propsToBeDeleted.setProperty(_, ""))
+    LogConfig.validateNames(propsToBeDeleted)
+    configsToBeDeleted
+  }
+
+  def parseReplicaAssignment(replicaAssignmentList: String): Map[Int, List[Int]] = {
+    val partitionList = replicaAssignmentList.split(",")
+    val ret = new mutable.LinkedHashMap[Int, List[Int]]()
+    for (i <- 0 until partitionList.size) {
+      val brokerList = partitionList(i).split(":").map(s => s.trim().toInt)
+      val duplicateBrokers = CoreUtils.duplicates(brokerList)
+      if (duplicateBrokers.nonEmpty)
+        throw new AdminCommandFailedException(s"Partition replica lists may not contain duplicate entries: ${duplicateBrokers.mkString(",")}")
+      ret.put(i, brokerList.toList)
+      if (ret(i).size != ret(0).size)
+        throw new AdminOperationException("Partition " + i + " has different replication factor: " + brokerList)
+    }
+    ret
+  }
+
+  def asJavaReplicaReassignment(original: Map[Int, List[Int]]): util.Map[Integer, util.List[Integer]] = {
+    original.map(f => Integer.valueOf(f._1) -> f._2.map(e => Integer.valueOf(e)).asJava).asJava
+  }
+
+  private def getReplicationFactor(tpi: TopicPartitionInfo, reassignment: Option[PartitionReassignment]): Int = {
+    // It is possible for a reassignment to complete between the time we have fetched its state and the time
+    // we fetch partition metadata. In ths case, we ignore the reassignment when determining replication factor.
+    def isReassignmentInProgress(ra: PartitionReassignment): Boolean = {
+      // Reassignment is still in progress as long as the removing replicas are still present
+      val allReplicaIds = tpi.replicas.asScala.map(_.id).toSet
+      val removingReplicaIds = ra.removingReplicas.asScala.map(Int.unbox).toSet
+      allReplicaIds.exists(removingReplicaIds.contains)
+    }
+
+    reassignment match {
+      case Some(ra) if isReassignmentInProgress(ra) => ra.replicas.asScala.diff(ra.addingReplicas.asScala).size
+      case _=> tpi.replicas.size
+    }
+  }
+
+  class TopicCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    private val bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: The Kafka server to connect to. In case of providing this, a direct Zookeeper connection won't be required.")
+      .withRequiredArg
+      .describedAs("server to connect to")
+      .ofType(classOf[String])
+    private val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client. " +
+      "This is used only with --bootstrap-server option for describing and altering broker configs.")
+      .withRequiredArg
+      .describedAs("command config property file")
+      .ofType(classOf[String])
+    private val zkConnectOpt = parser.accepts("zookeeper", "DEPRECATED, The connection string for the zookeeper connection in the form host:port. " +
+      "Multiple hosts can be given to allow fail-over.")
+      .withRequiredArg
+      .describedAs("hosts")
+      .ofType(classOf[String])
+    private val listOpt = parser.accepts("list", "List all available topics.")
+    private val createOpt = parser.accepts("create", "Create a new topic.")
+    private val deleteOpt = parser.accepts("delete", "Delete a topic")
+    private val alterOpt = parser.accepts("alter", "Alter the number of partitions, replica assignment, and/or configuration for the topic.")
+    private val describeOpt = parser.accepts("describe", "List details for the given topics.")
+    private val topicOpt = parser.accepts("topic", "The topic to create, alter, describe or delete. It also accepts a regular " +
+                                           "expression, except for --create option. Put topic name in double quotes and use the '\\' prefix " +
+                                           "to escape regular expression symbols; e.g. \"test\\.topic\".")
+                         .withRequiredArg
+                         .describedAs("topic")
+                         .ofType(classOf[String])
+    private val nl = System.getProperty("line.separator")
+    private val configOpt = parser.accepts("config", "A topic configuration override for the topic being created or altered."  +
+                                             "The following is a list of valid configurations: " + nl + LogConfig.configNames.map("\t" + _).mkString(nl) + nl +
+                                             "See the Kafka documentation for full details on the topic configs." +
+                                             "It is supported only in combination with --create if --bootstrap-server option is used.")
+                           .withRequiredArg
+                           .describedAs("name=value")
+                           .ofType(classOf[String])
+    private val deleteConfigOpt = parser.accepts("delete-config", "A topic configuration override to be removed for an existing topic (see the list of configurations under the --config option). " +
+      "Not supported with the --bootstrap-server option.")
+                           .withRequiredArg
+                           .describedAs("name")
+                           .ofType(classOf[String])
+    private val partitionsOpt = parser.accepts("partitions", "The number of partitions for the topic being created or " +
+      "altered (WARNING: If partitions are increased for a topic that has a key, the partition logic or ordering of the messages will be affected). If not supplied for create, defaults to the cluster default.")
+                           .withRequiredArg
+                           .describedAs("# of partitions")
+                           .ofType(classOf[java.lang.Integer])
+    private val replicationFactorOpt = parser.accepts("replication-factor", "The replication factor for each partition in the topic being created. If not supplied, defaults to the cluster default.")
+                           .withRequiredArg
+                           .describedAs("replication factor")
+                           .ofType(classOf[java.lang.Integer])
+    private val replicaAssignmentOpt = parser.accepts("replica-assignment", "A list of manual partition-to-broker assignments for the topic being created or altered.")
+                           .withRequiredArg
+                           .describedAs("broker_id_for_part1_replica1 : broker_id_for_part1_replica2 , " +
+                                        "broker_id_for_part2_replica1 : broker_id_for_part2_replica2 , ...")
+                           .ofType(classOf[String])
+    private val reportUnderReplicatedPartitionsOpt = parser.accepts("under-replicated-partitions",
+      "if set when describing topics, only show under replicated partitions")
+    private val reportUnavailablePartitionsOpt = parser.accepts("unavailable-partitions",
+      "if set when describing topics, only show partitions whose leader is not available")
+    private val reportUnderMinIsrPartitionsOpt = parser.accepts("under-min-isr-partitions",
+      "if set when describing topics, only show partitions whose isr count is less than the configured minimum. Not supported with the --zookeeper option.")
+    private val reportAtMinIsrPartitionsOpt = parser.accepts("at-min-isr-partitions",
+      "if set when describing topics, only show partitions whose isr count is equal to the configured minimum. Not supported with the --zookeeper option.")
+    private val topicsWithOverridesOpt = parser.accepts("topics-with-overrides",
+      "if set when describing topics, only show topics that have overridden configs")
+    private val ifExistsOpt = parser.accepts("if-exists",
+      "if set when altering or deleting or describing topics, the action will only execute if the topic exists. Not supported with the --bootstrap-server option.")
+    private val ifNotExistsOpt = parser.accepts("if-not-exists",
+      "if set when creating topics, the action will only execute if the topic does not already exist. Not supported with the --bootstrap-server option.")
+
+    private val disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment")
+
+    // This is not currently used, but we keep it for compatibility
+    parser.accepts("force", "Suppress console prompts")
+
+    private val excludeInternalTopicOpt = parser.accepts("exclude-internal",
+      "exclude internal topics when running list or describe command. The internal topics will be listed by default")
+
+    options = parser.parse(args : _*)
+
+    private val allTopicLevelOpts: Set[OptionSpec[_]] = Set(alterOpt, createOpt, describeOpt, listOpt, deleteOpt)
+
+    private val allReplicationReportOpts: Set[OptionSpec[_]] = Set(reportUnderReplicatedPartitionsOpt, reportUnderMinIsrPartitionsOpt, reportAtMinIsrPartitionsOpt, reportUnavailablePartitionsOpt)
+
+    def has(builder: OptionSpec[_]): Boolean = options.has(builder)
+    def valueAsOption[A](option: OptionSpec[A], defaultValue: Option[A] = None): Option[A] = if (has(option)) Some(options.valueOf(option)) else defaultValue
+    def valuesAsOption[A](option: OptionSpec[A], defaultValue: Option[util.List[A]] = None): Option[util.List[A]] = if (has(option)) Some(options.valuesOf(option)) else defaultValue
+
+    def hasCreateOption: Boolean = has(createOpt)
+    def hasAlterOption: Boolean = has(alterOpt)
+    def hasListOption: Boolean = has(listOpt)
+    def hasDescribeOption: Boolean = has(describeOpt)
+    def hasDeleteOption: Boolean = has(deleteOpt)
+
+    def zkConnect: Option[String] = valueAsOption(zkConnectOpt)
+    def bootstrapServer: Option[String] = valueAsOption(bootstrapServerOpt)
+    def commandConfig: Properties = if (has(commandConfigOpt)) Utils.loadProps(options.valueOf(commandConfigOpt)) else new Properties()
+    def topic: Option[String] = valueAsOption(topicOpt)
+    def partitions: Option[Integer] = valueAsOption(partitionsOpt)
+    def replicationFactor: Option[Integer] = valueAsOption(replicationFactorOpt)
+    def replicaAssignment: Option[Map[Int, List[Int]]] =
+      if (has(replicaAssignmentOpt) && !Option(options.valueOf(replicaAssignmentOpt)).getOrElse("").isEmpty)
+        Some(parseReplicaAssignment(options.valueOf(replicaAssignmentOpt)))
+      else
+        None
+    def rackAwareMode: RackAwareMode = if (has(disableRackAware)) RackAwareMode.Disabled else RackAwareMode.Enforced
+    def reportUnderReplicatedPartitions: Boolean = has(reportUnderReplicatedPartitionsOpt)
+    def reportUnavailablePartitions: Boolean = has(reportUnavailablePartitionsOpt)
+    def reportUnderMinIsrPartitions: Boolean = has(reportUnderMinIsrPartitionsOpt)
+    def reportAtMinIsrPartitions: Boolean = has(reportAtMinIsrPartitionsOpt)
+    def reportOverriddenConfigs: Boolean = has(topicsWithOverridesOpt)
+    def ifExists: Boolean = has(ifExistsOpt)
+    def ifNotExists: Boolean = has(ifNotExistsOpt)
+    def excludeInternalTopics: Boolean = has(excludeInternalTopicOpt)
+    def topicConfig: Option[util.List[String]] = valuesAsOption(configOpt)
+    def configsToDelete: Option[util.List[String]] = valuesAsOption(deleteConfigOpt)
+
+    def checkArgs(): Unit = {
+      if (args.length == 0)
+        CommandLineUtils.printUsageAndDie(parser, "Create, delete, describe, or change a topic.")
+
+      CommandLineUtils.printHelpAndExitIfNeeded(this, "This tool helps to create, delete, describe, or change a topic.")
+
+      // should have exactly one action
+      val actions = Seq(createOpt, listOpt, alterOpt, describeOpt, deleteOpt).count(options.has)
+      if (actions != 1)
+        CommandLineUtils.printUsageAndDie(parser, "Command must include exactly one action: --list, --describe, --create, --alter or --delete")
+
+      // check required args
+      if (has(bootstrapServerOpt) == has(zkConnectOpt))
+        throw new IllegalArgumentException("Only one of --bootstrap-server or --zookeeper must be specified")
+
+      if (!has(bootstrapServerOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, zkConnectOpt)
+      if(has(describeOpt) && has(ifExistsOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, topicOpt)
+      if (!has(listOpt) && !has(describeOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, topicOpt)
+      if (has(createOpt) && !has(replicaAssignmentOpt) && has(zkConnectOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, partitionsOpt, replicationFactorOpt)
+      if (has(bootstrapServerOpt) && has(alterOpt)) {
+        CommandLineUtils.checkInvalidArgsSet(parser, options, Set(bootstrapServerOpt, configOpt), Set(alterOpt))
+        CommandLineUtils.checkRequiredArgs(parser, options, partitionsOpt)
+      }
+
+      // check invalid args
+      CommandLineUtils.checkInvalidArgs(parser, options, configOpt, allTopicLevelOpts -- Set(alterOpt, createOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, deleteConfigOpt, allTopicLevelOpts -- Set(alterOpt) ++ Set(bootstrapServerOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, partitionsOpt, allTopicLevelOpts -- Set(alterOpt, createOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, replicationFactorOpt, allTopicLevelOpts -- Set(createOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, replicaAssignmentOpt, allTopicLevelOpts -- Set(createOpt,alterOpt))
+      if(options.has(createOpt))
+          CommandLineUtils.checkInvalidArgs(parser, options, replicaAssignmentOpt, Set(partitionsOpt, replicationFactorOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, reportUnderReplicatedPartitionsOpt,
+        allTopicLevelOpts -- Set(describeOpt) ++ allReplicationReportOpts - reportUnderReplicatedPartitionsOpt + topicsWithOverridesOpt)
+      CommandLineUtils.checkInvalidArgs(parser, options, reportUnderMinIsrPartitionsOpt,
+        allTopicLevelOpts -- Set(describeOpt) ++ allReplicationReportOpts - reportUnderMinIsrPartitionsOpt + topicsWithOverridesOpt + zkConnectOpt)
+      CommandLineUtils.checkInvalidArgs(parser, options, reportAtMinIsrPartitionsOpt,
+        allTopicLevelOpts -- Set(describeOpt) ++ allReplicationReportOpts - reportAtMinIsrPartitionsOpt + topicsWithOverridesOpt + zkConnectOpt)
+      CommandLineUtils.checkInvalidArgs(parser, options, reportUnavailablePartitionsOpt,
+        allTopicLevelOpts -- Set(describeOpt) ++ allReplicationReportOpts - reportUnavailablePartitionsOpt + topicsWithOverridesOpt)
+      CommandLineUtils.checkInvalidArgs(parser, options, topicsWithOverridesOpt,
+        allTopicLevelOpts -- Set(describeOpt) ++ allReplicationReportOpts)
+      CommandLineUtils.checkInvalidArgs(parser, options, ifExistsOpt, allTopicLevelOpts -- Set(alterOpt, deleteOpt, describeOpt) ++ Set(bootstrapServerOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, ifNotExistsOpt, allTopicLevelOpts -- Set(createOpt) ++ Set(bootstrapServerOpt))
+      CommandLineUtils.checkInvalidArgs(parser, options, excludeInternalTopicOpt, allTopicLevelOpts -- Set(listOpt, describeOpt))
+    }
+  }
+
+  def askToProceed(): Unit = {
+    println("Are you sure you want to continue? [y/n]")
+    if (!StdIn.readLine().equalsIgnoreCase("y")) {
+      println("Ending your session")
+      Exit.exit(0)
+    }
+  }
+
+}
+

core/src/main/scala/kafka/admin/ZkSecurityMigrator.scala

@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.admin
+
+import joptsimple.{ArgumentAcceptingOptionSpec, OptionSet}
+import kafka.server.KafkaConfig
+import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Exit, Logging}
+import kafka.zk.{ControllerZNode, KafkaZkClient, ZkData, ZkSecurityMigratorUtils}
+import org.apache.kafka.common.security.JaasUtils
+import org.apache.kafka.common.utils.{Time, Utils}
+import org.apache.zookeeper.AsyncCallback.{ChildrenCallback, StatCallback}
+import org.apache.zookeeper.KeeperException
+import org.apache.zookeeper.KeeperException.Code
+import org.apache.zookeeper.client.ZKClientConfig
+import org.apache.zookeeper.data.Stat
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+import scala.collection.mutable.Queue
+import scala.concurrent._
+import scala.concurrent.duration._
+
+/**
+ * This tool is to be used when making access to ZooKeeper authenticated or 
+ * the other way around, when removing authenticated access. The exact steps
+ * to migrate a Kafka cluster from unsecure to secure with respect to ZooKeeper
+ * access are the following:
+ * 
+ * 1- Perform a rolling upgrade of Kafka servers, setting zookeeper.set.acl to false
+ * and passing a valid JAAS login file via the system property 
+ * java.security.auth.login.config
+ * 2- Perform a second rolling upgrade keeping the system property for the login file
+ * and now setting zookeeper.set.acl to true
+ * 3- Finally run this tool. There is a script under ./bin. Run 
+ *   ./bin/zookeeper-security-migration.sh --help
+ * to see the configuration parameters. An example of running it is the following:
+ *  ./bin/zookeeper-security-migration.sh --zookeeper.acl=secure --zookeeper.connect=localhost:2181
+ * 
+ * To convert a cluster from secure to unsecure, we need to perform the following
+ * steps:
+ * 1- Perform a rolling upgrade setting zookeeper.set.acl to false for each server
+ * 2- Run this migration tool, setting zookeeper.acl to unsecure
+ * 3- Perform another rolling upgrade to remove the system property setting the
+ * login file (java.security.auth.login.config).
+ */
+
+object ZkSecurityMigrator extends Logging {
+  val usageMessage = ("ZooKeeper Migration Tool Help. This tool updates the ACLs of "
+                      + "znodes as part of the process of setting up ZooKeeper "
+                      + "authentication.")
+  val tlsConfigFileOption = "zk-tls-config-file"
+
+  def run(args: Array[String]): Unit = {
+    val jaasFile = System.getProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM)
+    val opts = new ZkSecurityMigratorOptions(args)
+
+    CommandLineUtils.printHelpAndExitIfNeeded(opts, usageMessage)
+
+    // Must have either SASL or TLS mutual authentication enabled to use this tool.
+    // Instantiate the client config we will use so that we take into account config provided via the CLI option
+    // and system properties passed via -D parameters if no CLI option is given.
+    val zkClientConfig = createZkClientConfigFromOption(opts.options, opts.zkTlsConfigFile).getOrElse(new ZKClientConfig())
+    val tlsClientAuthEnabled = KafkaConfig.zkTlsClientAuthEnabled(zkClientConfig)
+    if (jaasFile == null && !tlsClientAuthEnabled) {
+      val errorMsg = s"No JAAS configuration file has been specified and no TLS client certificate has been specified. Please make sure that you set " +
+        s"the system property ${JaasUtils.JAVA_LOGIN_CONFIG_PARAM} or provide a ZooKeeper client TLS configuration via --$tlsConfigFileOption <filename> " +
+        s"identifying at least ${KafkaConfig.ZkSslClientEnableProp}, ${KafkaConfig.ZkClientCnxnSocketProp}, and ${KafkaConfig.ZkSslKeyStoreLocationProp}"
+      System.err.println("ERROR: %s".format(errorMsg))
+      throw new IllegalArgumentException("Incorrect configuration")
+    }
+
+    if (!tlsClientAuthEnabled && !JaasUtils.isZkSaslEnabled()) {
+      val errorMsg = "Security isn't enabled, most likely the file isn't set properly: %s".format(jaasFile)
+      System.out.println("ERROR: %s".format(errorMsg))
+      throw new IllegalArgumentException("Incorrect configuration")
+    }
+
+    val zkAcl: Boolean = opts.options.valueOf(opts.zkAclOpt) match {
+      case "secure" =>
+        info("zookeeper.acl option is secure")
+        true
+      case "unsecure" =>
+        info("zookeeper.acl option is unsecure")
+        false
+      case _ =>
+        CommandLineUtils.printUsageAndDie(opts.parser, usageMessage)
+    }
+    val zkUrl = opts.options.valueOf(opts.zkUrlOpt)
+    val zkSessionTimeout = opts.options.valueOf(opts.zkSessionTimeoutOpt).intValue
+    val zkConnectionTimeout = opts.options.valueOf(opts.zkConnectionTimeoutOpt).intValue
+    val zkClient = KafkaZkClient(zkUrl, zkAcl, zkSessionTimeout, zkConnectionTimeout,
+      Int.MaxValue, Time.SYSTEM, zkClientConfig = Some(zkClientConfig))
+    val enablePathCheck = opts.options.has(opts.enablePathCheckOpt)
+    val migrator = new ZkSecurityMigrator(zkClient)
+    migrator.run(enablePathCheck)
+  }
+
+  def main(args: Array[String]): Unit = {
+    try {
+      run(args)
+    } catch {
+        case e: Exception => {
+          e.printStackTrace()
+          // must exit with non-zero status so system tests will know we failed
+          Exit.exit(1)
+        }
+    }
+  }
+
+  def createZkClientConfigFromFile(filename: String) : ZKClientConfig = {
+    val zkTlsConfigFileProps = Utils.loadProps(filename, KafkaConfig.ZkSslConfigToSystemPropertyMap.keys.toList.asJava)
+    val zkClientConfig = new ZKClientConfig() // Initializes based on any system properties that have been set
+    // Now override any set system properties with explicitly-provided values from the config file
+    // Emit INFO logs due to camel-case property names encouraging mistakes -- help people see mistakes they make
+    info(s"Found ${zkTlsConfigFileProps.size()} ZooKeeper client configuration properties in file $filename")
+    zkTlsConfigFileProps.entrySet().asScala.foreach(entry => {
+      val key = entry.getKey.toString
+      info(s"Setting $key")
+      KafkaConfig.setZooKeeperClientProperty(zkClientConfig, key, entry.getValue.toString)
+    })
+    zkClientConfig
+  }
+
+  private[admin] def createZkClientConfigFromOption(options: OptionSet, option: ArgumentAcceptingOptionSpec[String]) : Option[ZKClientConfig] =
+    if (!options.has(option))
+      None
+    else
+      Some(createZkClientConfigFromFile(options.valueOf(option)))
+
+  class ZkSecurityMigratorOptions(args: Array[String]) extends CommandDefaultOptions(args) {
+    val zkAclOpt = parser.accepts("zookeeper.acl", "Indicates whether to make the Kafka znodes in ZooKeeper secure or unsecure."
+      + " The options are 'secure' and 'unsecure'").withRequiredArg().ofType(classOf[String])
+    val zkUrlOpt = parser.accepts("zookeeper.connect", "Sets the ZooKeeper connect string (ensemble). This parameter " +
+      "takes a comma-separated list of host:port pairs.").withRequiredArg().defaultsTo("localhost:2181").
+      ofType(classOf[String])
+    val zkSessionTimeoutOpt = parser.accepts("zookeeper.session.timeout", "Sets the ZooKeeper session timeout.").
+      withRequiredArg().ofType(classOf[java.lang.Integer]).defaultsTo(30000)
+    val zkConnectionTimeoutOpt = parser.accepts("zookeeper.connection.timeout", "Sets the ZooKeeper connection timeout.").
+      withRequiredArg().ofType(classOf[java.lang.Integer]).defaultsTo(30000)
+    val enablePathCheckOpt = parser.accepts("enable.path.check", "Checks if all the root paths exist in ZooKeeper " +
+      "before migration. If not, exit the command.")
+    val zkTlsConfigFile = parser.accepts(tlsConfigFileOption,
+      "Identifies the file where ZooKeeper client TLS connectivity properties are defined.  Any properties other than " +
+        KafkaConfig.ZkSslConfigToSystemPropertyMap.keys.mkString(", ") + " are ignored.")
+      .withRequiredArg().describedAs("ZooKeeper TLS configuration").ofType(classOf[String])
+    options = parser.parse(args : _*)
+  }
+}
+
+class ZkSecurityMigrator(zkClient: KafkaZkClient) extends Logging {
+  private val zkSecurityMigratorUtils = new ZkSecurityMigratorUtils(zkClient)
+  private val futures = new Queue[Future[String]]
+
+  private def setAcl(path: String, setPromise: Promise[String]): Unit = {
+    info("Setting ACL for path %s".format(path))
+    zkSecurityMigratorUtils.currentZooKeeper.setACL(path, zkClient.defaultAcls(path).asJava, -1, SetACLCallback, setPromise)
+  }
+
+  private def getChildren(path: String, childrenPromise: Promise[String]): Unit = {
+    info("Getting children to set ACLs for path %s".format(path))
+    zkSecurityMigratorUtils.currentZooKeeper.getChildren(path, false, GetChildrenCallback, childrenPromise)
+  }
+
+  private def setAclIndividually(path: String): Unit = {
+    val setPromise = Promise[String]
+    futures.synchronized {
+      futures += setPromise.future
+    }
+    setAcl(path, setPromise)
+  }
+
+  private def setAclsRecursively(path: String): Unit = {
+    val setPromise = Promise[String]
+    val childrenPromise = Promise[String]
+    futures.synchronized {
+      futures += setPromise.future
+      futures += childrenPromise.future
+    }
+    setAcl(path, setPromise)
+    getChildren(path, childrenPromise)
+  }
+
+  private object GetChildrenCallback extends ChildrenCallback {
+    def processResult(rc: Int,
+                      path: String,
+                      ctx: Object,
+                      children: java.util.List[String]): Unit = {
+      val zkHandle = zkSecurityMigratorUtils.currentZooKeeper
+      val promise = ctx.asInstanceOf[Promise[String]]
+      Code.get(rc) match {
+        case Code.OK =>
+          // Set ACL for each child
+          children.asScala.map { child =>
+            path match {
+              case "/" => s"/$child"
+              case path => s"$path/$child"
+            }
+          }.foreach(setAclsRecursively)
+          promise success "done"
+        case Code.CONNECTIONLOSS =>
+          zkHandle.getChildren(path, false, GetChildrenCallback, ctx)
+        case Code.NONODE =>
+          warn("Node is gone, it could be have been legitimately deleted: %s".format(path))
+          promise success "done"
+        case Code.SESSIONEXPIRED =>
+          // Starting a new session isn't really a problem, but it'd complicate
+          // the logic of the tool, so we quit and let the user re-run it.
+          System.out.println("ZooKeeper session expired while changing ACLs")
+          promise failure KeeperException.create(Code.get(rc))
+        case _ =>
+          System.out.println("Unexpected return code: %d".format(rc))
+          promise failure KeeperException.create(Code.get(rc))
+      }
+    }
+  }
+
+  private object SetACLCallback extends StatCallback {
+    def processResult(rc: Int,
+                      path: String,
+                      ctx: Object,
+                      stat: Stat): Unit = {
+      val zkHandle = zkSecurityMigratorUtils.currentZooKeeper
+      val promise = ctx.asInstanceOf[Promise[String]]
+
+      Code.get(rc) match {
+        case Code.OK =>
+          info("Successfully set ACLs for %s".format(path))
+          promise success "done"
+        case Code.CONNECTIONLOSS =>
+            zkHandle.setACL(path, zkClient.defaultAcls(path).asJava, -1, SetACLCallback, ctx)
+        case Code.NONODE =>
+          warn("Znode is gone, it could be have been legitimately deleted: %s".format(path))
+          promise success "done"
+        case Code.SESSIONEXPIRED =>
+          // Starting a new session isn't really a problem, but it'd complicate
+          // the logic of the tool, so we quit and let the user re-run it.
+          System.out.println("ZooKeeper session expired while changing ACLs")
+          promise failure KeeperException.create(Code.get(rc))
+        case _ =>
+          System.out.println("Unexpected return code: %d".format(rc))
+          promise failure KeeperException.create(Code.get(rc))
+      }
+    }
+  }
+
+  private def run(enablePathCheck: Boolean): Unit = {
+    try {
+      setAclIndividually("/")
+      checkPathExistenceAndMaybeExit(enablePathCheck)
+      for (path <- ZkData.SecureRootPaths) {
+        debug("Going to set ACL for %s".format(path))
+        if (path == ControllerZNode.path && !zkClient.pathExists(path)) {
+          debug("Ignoring to set ACL for %s, because it doesn't exist".format(path))
+        } else {
+          zkClient.makeSurePersistentPathExists(path)
+          setAclsRecursively(path)
+        }
+      }
+
+      @tailrec
+      def recurse(): Unit = {
+        val future = futures.synchronized { 
+          futures.headOption
+        }
+        future match {
+          case Some(a) =>
+            Await.result(a, 6000 millis)
+            futures.synchronized { futures.dequeue }
+            recurse
+          case None =>
+        }
+      }
+      recurse()
+
+    } finally {
+      zkClient.close
+    }
+  }
+
+  private def checkPathExistenceAndMaybeExit(enablePathCheck: Boolean): Unit = {
+    val nonExistingSecureRootPaths = ZkData.SecureRootPaths.filterNot(zkClient.pathExists)
+    if (nonExistingSecureRootPaths.nonEmpty) {
+      println(s"Warning: The following secure root paths do not exist in ZooKeeper: ${nonExistingSecureRootPaths.mkString(",")}")
+      println("That might be due to an incorrect chroot is specified when executing the command.")
+      if (enablePathCheck) {
+        println("Exit the command.")
+        // must exit with non-zero status so system tests will know we failed
+        Exit.exit(1)
+      }
+    }
+  }
+}

core/src/main/scala/kafka/api/ApiUtils.scala

@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.api
+
+import java.nio.ByteBuffer
+import java.nio.charset.StandardCharsets
+
+import org.apache.kafka.common.KafkaException
+
+/**
+ * Helper functions specific to parsing or serializing requests and responses
+ */
+object ApiUtils {
+
+    /**
+   * Read size prefixed string where the size is stored as a 2 byte short.
+   * @param buffer The buffer to read from
+   */
+  def readShortString(buffer: ByteBuffer): String = {
+    val size: Int = buffer.getShort()
+    if(size < 0)
+      return null
+    val bytes = new Array[Byte](size)
+    buffer.get(bytes)
+    new String(bytes, StandardCharsets.UTF_8)
+  }
+  
+  /**
+   * Write a size prefixed string where the size is stored as a 2 byte short
+   * @param buffer The buffer to write to
+   * @param string The string to write
+   */
+  def writeShortString(buffer: ByteBuffer, string: String): Unit = {
+    if(string == null) {
+      buffer.putShort(-1)
+    } else {
+      val encodedString = string.getBytes(StandardCharsets.UTF_8)
+      if(encodedString.length > Short.MaxValue) {
+        throw new KafkaException("String exceeds the maximum size of " + Short.MaxValue + ".")
+      } else {
+        buffer.putShort(encodedString.length.asInstanceOf[Short])
+        buffer.put(encodedString)
+      }
+    }
+  }
+  
+  /**
+   * Return size of a size prefixed string where the size is stored as a 2 byte short
+   * @param string The string to write
+   */
+  def shortStringLength(string: String): Int = {
+    if(string == null) {
+      2
+    } else {
+      val encodedString = string.getBytes(StandardCharsets.UTF_8)
+      if(encodedString.length > Short.MaxValue) {
+        throw new KafkaException("String exceeds the maximum size of " + Short.MaxValue + ".")
+      } else {
+        2 + encodedString.length
+      }
+    }
+  }
+  
+}

core/src/main/scala/kafka/api/ApiVersion.scala

@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.api
+
+import org.apache.kafka.common.config.ConfigDef.Validator
+import org.apache.kafka.common.config.ConfigException
+import org.apache.kafka.common.record.RecordVersion
+
+/**
+ * This class contains the different Kafka versions.
+ * Right now, we use them for upgrades - users can configure the version of the API brokers will use to communicate between themselves.
+ * This is only for inter-broker communications - when communicating with clients, the client decides on the API version.
+ *
+ * Note that the ID we initialize for each version is important.
+ * We consider a version newer than another, if it has a higher ID (to avoid depending on lexicographic order)
+ *
+ * Since the api protocol may change more than once within the same release and to facilitate people deploying code from
+ * trunk, we have the concept of internal versions (first introduced during the 0.10.0 development cycle). For example,
+ * the first time we introduce a version change in a release, say 0.10.0, we will add a config value "0.10.0-IV0" and a
+ * corresponding case object KAFKA_0_10_0-IV0. We will also add a config value "0.10.0" that will be mapped to the
+ * latest internal version object, which is KAFKA_0_10_0-IV0. When we change the protocol a second time while developing
+ * 0.10.0, we will add a new config value "0.10.0-IV1" and a corresponding case object KAFKA_0_10_0-IV1. We will change
+ * the config value "0.10.0" to map to the latest internal version object KAFKA_0_10_0-IV1. The config value of
+ * "0.10.0-IV0" is still mapped to KAFKA_0_10_0-IV0. This way, if people are deploying from trunk, they can use
+ * "0.10.0-IV0" and "0.10.0-IV1" to upgrade one internal version at a time. For most people who just want to use
+ * released version, they can use "0.10.0" when upgrading to the 0.10.0 release.
+ */
+object ApiVersion {
+  // This implicit is necessary due to: https://issues.scala-lang.org/browse/SI-8541
+  implicit def orderingByVersion[A <: ApiVersion]: Ordering[A] = Ordering.by(_.id)
+
+  val allVersions: Seq[ApiVersion] = Seq(
+    KAFKA_0_8_0,
+    KAFKA_0_8_1,
+    KAFKA_0_8_2,
+    KAFKA_0_9_0,
+    // 0.10.0-IV0 is introduced for KIP-31/32 which changes the message format.
+    KAFKA_0_10_0_IV0,
+    // 0.10.0-IV1 is introduced for KIP-36(rack awareness) and KIP-43(SASL handshake).
+    KAFKA_0_10_0_IV1,
+    // introduced for JoinGroup protocol change in KIP-62
+    KAFKA_0_10_1_IV0,
+    // 0.10.1-IV1 is introduced for KIP-74(fetch response size limit).
+    KAFKA_0_10_1_IV1,
+    // introduced ListOffsetRequest v1 in KIP-79
+    KAFKA_0_10_1_IV2,
+    // introduced UpdateMetadataRequest v3 in KIP-103
+    KAFKA_0_10_2_IV0,
+    // KIP-98 (idempotent and transactional producer support)
+    KAFKA_0_11_0_IV0,
+    // introduced DeleteRecordsRequest v0 and FetchRequest v4 in KIP-107
+    KAFKA_0_11_0_IV1,
+    // Introduced leader epoch fetches to the replica fetcher via KIP-101
+    KAFKA_0_11_0_IV2,
+    // Introduced LeaderAndIsrRequest V1, UpdateMetadataRequest V4 and FetchRequest V6 via KIP-112
+    KAFKA_1_0_IV0,
+    // Introduced DeleteGroupsRequest V0 via KIP-229, plus KIP-227 incremental fetch requests,
+    // and KafkaStorageException for fetch requests.
+    KAFKA_1_1_IV0,
+    // Introduced OffsetsForLeaderEpochRequest V1 via KIP-279 (Fix log divergence between leader and follower after fast leader fail over)
+    KAFKA_2_0_IV0,
+    // Several request versions were bumped due to KIP-219 (Improve quota communication)
+    KAFKA_2_0_IV1,
+    // Introduced new schemas for group offset (v2) and group metadata (v2) (KIP-211)
+    KAFKA_2_1_IV0,
+    // New Fetch, OffsetsForLeaderEpoch, and ListOffsets schemas (KIP-320)
+    KAFKA_2_1_IV1,
+    // Support ZStandard Compression Codec (KIP-110)
+    KAFKA_2_1_IV2,
+    // Introduced broker generation (KIP-380), and
+    // LeaderAdnIsrRequest V2, UpdateMetadataRequest V5, StopReplicaRequest V1
+    KAFKA_2_2_IV0,
+    // New error code for ListOffsets when a new leader is lagging behind former HW (KIP-207)
+    KAFKA_2_2_IV1,
+    // Introduced static membership.
+    KAFKA_2_3_IV0,
+    // Add rack_id to FetchRequest, preferred_read_replica to FetchResponse, and replica_id to OffsetsForLeaderRequest
+    KAFKA_2_3_IV1,
+    // Add adding_replicas and removing_replicas fields to LeaderAndIsrRequest
+    KAFKA_2_4_IV0,
+    // Flexible version support in inter-broker APIs
+    KAFKA_2_4_IV1,
+    // No new APIs, equivalent to 2.4-IV1
+    KAFKA_2_5_IV0
+  )
+
+  // Map keys are the union of the short and full versions
+  private val versionMap: Map[String, ApiVersion] =
+    allVersions.map(v => v.version -> v).toMap ++ allVersions.groupBy(_.shortVersion).map { case (k, v) => k -> v.last }
+
+  /**
+   * Return an `ApiVersion` instance for `versionString`, which can be in a variety of formats (e.g. "0.8.0", "0.8.0.x",
+   * "0.10.0", "0.10.0-IV1"). `IllegalArgumentException` is thrown if `versionString` cannot be mapped to an `ApiVersion`.
+   */
+  def apply(versionString: String): ApiVersion = {
+    val versionSegments = versionString.split('.').toSeq
+    val numSegments = if (versionString.startsWith("0.")) 3 else 2
+    val key = versionSegments.take(numSegments).mkString(".")
+    versionMap.getOrElse(key, throw new IllegalArgumentException(s"Version `$versionString` is not a valid version"))
+  }
+
+  def latestVersion: ApiVersion = allVersions.last
+
+  /**
+   * Return the minimum `ApiVersion` that supports `RecordVersion`.
+   */
+  def minSupportedFor(recordVersion: RecordVersion): ApiVersion = {
+    recordVersion match {
+      case RecordVersion.V0 => KAFKA_0_8_0
+      case RecordVersion.V1 => KAFKA_0_10_0_IV0
+      case RecordVersion.V2 => KAFKA_0_11_0_IV0
+      case _ => throw new IllegalArgumentException(s"Invalid message format version $recordVersion")
+    }
+  }
+}
+
+sealed trait ApiVersion extends Ordered[ApiVersion] {
+  def version: String
+  def shortVersion: String
+  def recordVersion: RecordVersion
+  def id: Int
+
+  override def compare(that: ApiVersion): Int =
+    ApiVersion.orderingByVersion.compare(this, that)
+
+  override def toString: String = version
+}
+
+/**
+ * For versions before 0.10.0, `version` and `shortVersion` were the same.
+ */
+sealed trait LegacyApiVersion extends ApiVersion {
+  def version = shortVersion
+}
+
+/**
+ * From 0.10.0 onwards, each version has a sub-version. For example, IV0 is the sub-version of 0.10.0-IV0.
+ */
+sealed trait DefaultApiVersion extends ApiVersion {
+  lazy val version = shortVersion + "-" + subVersion
+  protected def subVersion: String
+}
+
+// Keep the IDs in order of versions
+case object KAFKA_0_8_0 extends LegacyApiVersion {
+  val shortVersion = "0.8.0"
+  val recordVersion = RecordVersion.V0
+  val id: Int = 0
+}
+
+case object KAFKA_0_8_1 extends LegacyApiVersion {
+  val shortVersion = "0.8.1"
+  val recordVersion = RecordVersion.V0
+  val id: Int = 1
+}
+
+case object KAFKA_0_8_2 extends LegacyApiVersion {
+  val shortVersion = "0.8.2"
+  val recordVersion = RecordVersion.V0
+  val id: Int = 2
+}
+
+case object KAFKA_0_9_0 extends LegacyApiVersion {
+  val shortVersion = "0.9.0"
+  val subVersion = ""
+  val recordVersion = RecordVersion.V0
+  val id: Int = 3
+}
+
+case object KAFKA_0_10_0_IV0 extends DefaultApiVersion {
+  val shortVersion = "0.10.0"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V1
+  val id: Int = 4
+}
+
+case object KAFKA_0_10_0_IV1 extends DefaultApiVersion {
+  val shortVersion = "0.10.0"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V1
+  val id: Int = 5
+}
+
+case object KAFKA_0_10_1_IV0 extends DefaultApiVersion {
+  val shortVersion = "0.10.1"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V1
+  val id: Int = 6
+}
+
+case object KAFKA_0_10_1_IV1 extends DefaultApiVersion {
+  val shortVersion = "0.10.1"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V1
+  val id: Int = 7
+}
+
+case object KAFKA_0_10_1_IV2 extends DefaultApiVersion {
+  val shortVersion = "0.10.1"
+  val subVersion = "IV2"
+  val recordVersion = RecordVersion.V1
+  val id: Int = 8
+}
+
+case object KAFKA_0_10_2_IV0 extends DefaultApiVersion {
+  val shortVersion = "0.10.2"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V1
+  val id: Int = 9
+}
+
+case object KAFKA_0_11_0_IV0 extends DefaultApiVersion {
+  val shortVersion = "0.11.0"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 10
+}
+
+case object KAFKA_0_11_0_IV1 extends DefaultApiVersion {
+  val shortVersion = "0.11.0"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 11
+}
+
+case object KAFKA_0_11_0_IV2 extends DefaultApiVersion {
+  val shortVersion = "0.11.0"
+  val subVersion = "IV2"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 12
+}
+
+case object KAFKA_1_0_IV0 extends DefaultApiVersion {
+  val shortVersion = "1.0"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 13
+}
+
+case object KAFKA_1_1_IV0 extends DefaultApiVersion {
+  val shortVersion = "1.1"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 14
+}
+
+case object KAFKA_2_0_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.0"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 15
+}
+
+case object KAFKA_2_0_IV1 extends DefaultApiVersion {
+  val shortVersion: String = "2.0"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 16
+}
+
+case object KAFKA_2_1_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.1"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 17
+}
+
+case object KAFKA_2_1_IV1 extends DefaultApiVersion {
+  val shortVersion: String = "2.1"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 18
+}
+
+case object KAFKA_2_1_IV2 extends DefaultApiVersion {
+  val shortVersion: String = "2.1"
+  val subVersion = "IV2"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 19
+}
+
+case object KAFKA_2_2_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.2"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 20
+}
+
+case object KAFKA_2_2_IV1 extends DefaultApiVersion {
+  val shortVersion: String = "2.2"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 21
+}
+
+case object KAFKA_2_3_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.3"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 22
+}
+
+case object KAFKA_2_3_IV1 extends DefaultApiVersion {
+  val shortVersion: String = "2.3"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 23
+}
+
+case object KAFKA_2_4_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.4"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 24
+}
+
+case object KAFKA_2_4_IV1 extends DefaultApiVersion {
+  val shortVersion: String = "2.4"
+  val subVersion = "IV1"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 25
+}
+
+case object KAFKA_2_5_IV0 extends DefaultApiVersion {
+  val shortVersion: String = "2.5"
+  val subVersion = "IV0"
+  val recordVersion = RecordVersion.V2
+  val id: Int = 26
+}
+
+object ApiVersionValidator extends Validator {
+
+  override def ensureValid(name: String, value: Any): Unit = {
+    try {
+      ApiVersion(value.toString)
+    } catch {
+      case e: IllegalArgumentException => throw new ConfigException(name, value.toString, e.getMessage)
+    }
+  }
+
+  override def toString: String = "[" + ApiVersion.allVersions.map(_.version).distinct.mkString(", ") + "]"
+}

core/src/main/scala/kafka/api/LeaderAndIsr.scala

@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.api
+
+import com.didichuxing.datachannel.kafka.config.manager.TopicConfigManager
+import org.apache.kafka.common.TopicPartition
+
+object LeaderAndIsr {
+  val initialLeaderEpoch: Int = 0
+  val initialZKVersion: Int = 0
+  val NoLeader: Int = -1
+  val LeaderDuringDelete: Int = -2
+
+  def apply(leader: Int, isr: List[Int]): LeaderAndIsr = LeaderAndIsr(leader, initialLeaderEpoch, isr, initialZKVersion)
+
+  def duringDelete(isr: List[Int]): LeaderAndIsr = LeaderAndIsr(LeaderDuringDelete, isr)
+}
+
+case class LeaderAndIsr(leader: Int,
+                        leaderEpoch: Int,
+                        isr: List[Int],
+                        zkVersion: Int) {
+  def withZkVersion(zkVersion: Int): LeaderAndIsr = copy(zkVersion = zkVersion)
+
+  def newLeader(leader: Int) = newLeaderAndIsr(leader, isr)
+
+  def newLeaderAndIsr(leader: Int, isr: List[Int], holdEpoch: Boolean = false): LeaderAndIsr = LeaderAndIsr(leader, if (holdEpoch) leaderEpoch else leaderEpoch + 1, isr, zkVersion)
+
+  def newEpochAndZkVersion = newLeaderAndIsr(leader, isr)
+
+
+  def newLeader(partition: TopicPartition, leader: Int): LeaderAndIsr = newLeaderAndIsr(partition, leader, isr)
+
+  def newLeaderAndIsr(partition: TopicPartition, leader: Int, isr: List[Int]): LeaderAndIsr = newLeaderAndIsr(leader, isr, isHoldEpoch(partition))
+
+  def newEpochAndZkVersion(partition: TopicPartition): LeaderAndIsr = newLeaderAndIsr(partition, leader, isr)
+
+  private def isHoldEpoch(partition: TopicPartition): Boolean = {
+    TopicConfigManager.isMirrorTopicByLocal(partition.topic())
+  }
+
+  def leaderOpt: Option[Int] = {
+    if (leader == LeaderAndIsr.NoLeader) None else Some(leader)
+  }
+
+  override def toString: String = {
+    s"LeaderAndIsr(leader=$leader, leaderEpoch=$leaderEpoch, isr=$isr, zkVersion=$zkVersion)"
+  }
+}

core/src/main/scala/kafka/api/Request.scala

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.api
+
+object Request {
+  val OrdinaryConsumerId: Int = -1
+  val DebuggingConsumerId: Int = -2
+  val FutureLocalReplicaId: Int = -3
+
+  // Broker ids are non-negative int.
+  def isValidBrokerId(brokerId: Int): Boolean = brokerId >= 0
+
+  def describeReplicaId(replicaId: Int): String = {
+    replicaId match {
+      case OrdinaryConsumerId => "consumer"
+      case DebuggingConsumerId => "debug consumer"
+      case FutureLocalReplicaId => "future local replica"
+      case id if isValidBrokerId(id) => s"replica [$id]"
+      case id => s"invalid replica [$id]"
+    }
+  }
+}

core/src/main/scala/kafka/api/package.scala

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka
+
+import org.apache.kafka.common.ElectionType
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.requests.ElectLeadersRequest
+import scala.collection.JavaConverters._
+
+package object api {
+  implicit final class ElectLeadersRequestOps(val self: ElectLeadersRequest) extends AnyVal {
+    def topicPartitions: Set[TopicPartition] = {
+      if (self.data.topicPartitions == null) {
+        Set.empty
+      } else {
+        self.data.topicPartitions.asScala.iterator.flatMap { topicPartition =>
+          topicPartition.partitionId.asScala.map { partitionId =>
+            new TopicPartition(topicPartition.topic, partitionId)
+          }
+        }.toSet
+      }
+    }
+
+    def electionType: ElectionType = {
+      if (self.version == 0) {
+        ElectionType.PREFERRED
+      } else {
+        ElectionType.valueOf(self.data.electionType)
+      }
+    }
+  }
+}

core/src/main/scala/kafka/cluster/Broker.scala

@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.cluster
+
+import java.util
+
+import kafka.common.BrokerEndPointNotAvailableException
+import kafka.server.KafkaConfig
+import org.apache.kafka.common.{ClusterResource, Endpoint, Node}
+import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.apache.kafka.server.authorizer.AuthorizerServerInfo
+
+import scala.collection.Seq
+import scala.collection.JavaConverters._
+
+object Broker {
+  private[cluster] case class ServerInfo(clusterResource: ClusterResource,
+                                         brokerId: Int,
+                                         endpoints: util.List[Endpoint],
+                                         interBrokerEndpoint: Endpoint) extends AuthorizerServerInfo
+}
+
+/**
+ * A Kafka broker.
+ * A broker has an id, a collection of end-points, an optional rack and a listener to security protocol map.
+ * Each end-point is (host, port, listenerName).
+ */
+case class Broker(id: Int, endPoints: Seq[EndPoint], rack: Option[String]) {
+
+  private val endPointsMap = endPoints.map { endPoint =>
+    endPoint.listenerName -> endPoint
+  }.toMap
+
+  if (endPointsMap.size != endPoints.size)
+    throw new IllegalArgumentException(s"There is more than one end point with the same listener name: ${endPoints.mkString(",")}")
+
+  override def toString: String =
+    s"$id : ${endPointsMap.values.mkString("(",",",")")} : ${rack.orNull}"
+
+  def this(id: Int, host: String, port: Int, listenerName: ListenerName, protocol: SecurityProtocol) = {
+    this(id, Seq(EndPoint(host, port, listenerName, protocol)), None)
+  }
+
+  def this(bep: BrokerEndPoint, listenerName: ListenerName, protocol: SecurityProtocol) = {
+    this(bep.id, bep.host, bep.port, listenerName, protocol)
+  }
+
+  def node(listenerName: ListenerName): Node =
+    getNode(listenerName).getOrElse {
+      throw new BrokerEndPointNotAvailableException(s"End point with listener name ${listenerName.value} not found " +
+        s"for broker $id")
+    }
+
+  def getNode(listenerName: ListenerName): Option[Node] =
+    endPointsMap.get(listenerName).map(endpoint => new Node(id, endpoint.host, endpoint.port, rack.orNull))
+
+  def brokerEndPoint(listenerName: ListenerName): BrokerEndPoint = {
+    val endpoint = endPoint(listenerName)
+    new BrokerEndPoint(id, endpoint.host, endpoint.port)
+  }
+
+  def endPoint(listenerName: ListenerName): EndPoint = {
+    endPointsMap.getOrElse(listenerName,
+      throw new BrokerEndPointNotAvailableException(s"End point with listener name ${listenerName.value} not found for broker $id"))
+  }
+
+  def toServerInfo(clusterId: String, config: KafkaConfig): AuthorizerServerInfo = {
+    val clusterResource: ClusterResource = new ClusterResource(clusterId)
+    val interBrokerEndpoint: Endpoint = endPoint(config.interBrokerListenerName).toJava
+    val brokerEndpoints: util.List[Endpoint] = endPoints.toList.map(_.toJava).asJava
+    Broker.ServerInfo(clusterResource, id, brokerEndpoints, interBrokerEndpoint)
+  }
+}

core/src/main/scala/kafka/cluster/BrokerEndPoint.scala

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.cluster
+
+
+import kafka.api.ApiUtils._
+import org.apache.kafka.common.KafkaException
+import org.apache.kafka.common.utils.Utils._
+
+import java.nio.ByteBuffer
+
+object BrokerEndPoint {
+
+  private val uriParseExp = """\[?([0-9a-zA-Z\-%._:]*)\]?:([0-9]+)""".r
+
+  /**
+   * BrokerEndPoint URI is host:port or [ipv6_host]:port
+   * Note that unlike EndPoint (or listener) this URI has no security information.
+   */
+  def parseHostPort(connectionString: String): Option[(String, Int)] = {
+    connectionString match {
+      case uriParseExp(host, port) => try Some(host, port.toInt) catch { case _: NumberFormatException => None }
+      case _ => None
+    }
+  }
+  
+  /**
+   * BrokerEndPoint URI is host:port or [ipv6_host]:port
+   * Note that unlike EndPoint (or listener) this URI has no security information.
+   */
+  def createBrokerEndPoint(brokerId: Int, connectionString: String): BrokerEndPoint = {
+    parseHostPort(connectionString).map { case (host, port) => new BrokerEndPoint(brokerId, host, port) }.getOrElse {
+      throw new KafkaException("Unable to parse " + connectionString + " to a broker endpoint")
+    }
+  }
+
+  def readFrom(buffer: ByteBuffer): BrokerEndPoint = {
+    val brokerId = buffer.getInt()
+    val host = readShortString(buffer)
+    val port = buffer.getInt()
+    BrokerEndPoint(brokerId, host, port)
+  }
+}
+
+/**
+ * BrokerEndpoint is used to connect to specific host:port pair.
+ * It is typically used by clients (or brokers when connecting to other brokers)
+ * and contains no information about the security protocol used on the connection.
+ * Clients should know which security protocol to use from configuration.
+ * This allows us to keep the wire protocol with the clients unchanged where the protocol is not needed.
+ */
+case class BrokerEndPoint(id: Int, host: String, port: Int, remoteCluster: Option[String] = None) {
+
+  def connectionString(): String = formatAddress(host, port)
+
+  def writeTo(buffer: ByteBuffer): Unit = {
+    buffer.putInt(id)
+    writeShortString(buffer, host)
+    buffer.putInt(port)
+  }
+
+  def sizeInBytes: Int =
+    4 + /* broker Id */
+    4 + /* port */
+    shortStringLength(host)
+
+  override def toString: String = {
+    s"BrokerEndPoint(id=$id, host=$host:$port)"
+  }
+}

core/src/main/scala/kafka/cluster/Cluster.scala

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.cluster
+
+import scala.collection._
+
+/**
+ * The set of active brokers in the cluster
+ */
+private[kafka] class Cluster {
+  
+  private val brokers = new mutable.HashMap[Int, Broker]
+  
+  def this(brokerList: Iterable[Broker]) {
+    this()
+	  for(broker <- brokerList)
+      brokers.put(broker.id, broker)
+  }
+
+  def getBroker(id: Int): Option[Broker] = brokers.get(id)
+  
+  def add(broker: Broker) = brokers.put(broker.id, broker)
+  
+  def remove(id: Int) = brokers.remove(id)
+  
+  def size = brokers.size
+  
+  override def toString: String =
+    "Cluster(" + brokers.values.mkString(", ") + ")"  
+}

core/src/main/scala/kafka/cluster/EndPoint.scala

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.cluster
+
+import org.apache.kafka.common.{Endpoint => JEndpoint, KafkaException}
+import org.apache.kafka.common.network.ListenerName
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.apache.kafka.common.utils.Utils
+
+import scala.collection.Map
+
+object EndPoint {
+
+  private val uriParseExp = """^(.*)://\[?([0-9a-zA-Z\-%._:]*)\]?:(-?[0-9]+)""".r
+
+  private[kafka] val DefaultSecurityProtocolMap: Map[ListenerName, SecurityProtocol] =
+    SecurityProtocol.values.map(sp => ListenerName.forSecurityProtocol(sp) -> sp).toMap
+
+  /**
+   * Create EndPoint object from `connectionString` and optional `securityProtocolMap`. If the latter is not provided,
+   * we fallback to the default behaviour where listener names are the same as security protocols.
+   *
+   * @param connectionString the format is listener_name://host:port or listener_name://[ipv6 host]:port
+   *                         for example: PLAINTEXT://myhost:9092, CLIENT://myhost:9092 or REPLICATION://[::1]:9092
+   *                         Host can be empty (PLAINTEXT://:9092) in which case we'll bind to default interface
+   *                         Negative ports are also accepted, since they are used in some unit tests
+   */
+  def createEndPoint(connectionString: String, securityProtocolMap: Option[Map[ListenerName, SecurityProtocol]]): EndPoint = {
+    val protocolMap = securityProtocolMap.getOrElse(DefaultSecurityProtocolMap)
+
+    def securityProtocol(listenerName: ListenerName): SecurityProtocol =
+      protocolMap.getOrElse(listenerName,
+        throw new IllegalArgumentException(s"No security protocol defined for listener ${listenerName.value}"))
+
+    connectionString match {
+      case uriParseExp(listenerNameString, "", port) =>
+        val listenerName = ListenerName.normalised(listenerNameString)
+        new EndPoint(null, port.toInt, listenerName, securityProtocol(listenerName))
+      case uriParseExp(listenerNameString, host, port) =>
+        val listenerName = ListenerName.normalised(listenerNameString)
+        new EndPoint(host, port.toInt, listenerName, securityProtocol(listenerName))
+      case _ => throw new KafkaException(s"Unable to parse $connectionString to a broker endpoint")
+    }
+  }
+}
+
+/**
+ * Part of the broker definition - matching host/port pair to a protocol
+ */
+case class EndPoint(host: String, port: Int, listenerName: ListenerName, securityProtocol: SecurityProtocol) {
+  def connectionString: String = {
+    val hostport =
+      if (host == null)
+        ":"+port
+      else
+        Utils.formatAddress(host, port)
+    listenerName.value + "://" + hostport
+  }
+
+  def toJava: JEndpoint = {
+    new JEndpoint(listenerName.value, securityProtocol, host, port)
+  }
+}

core/src/main/scala/kafka/cluster/Replica.scala

@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package kafka.cluster
+
+import kafka.log.{Log}
+import kafka.utils.Logging
+import kafka.server.{LogOffsetMetadata}
+import org.apache.kafka.common.{TopicPartition}
+
+class Replica(val brokerId: Int, val topicPartition: TopicPartition) extends Logging {
+  // the log end offset value, kept in all replicas;
+  // for local replica it is the log's end offset, for remote replicas its value is only updated by follower fetch
+  @volatile private[this] var _logEndOffsetMetadata = LogOffsetMetadata.UnknownOffsetMetadata
+  // the log start offset value, kept in all replicas;
+  // for local replica it is the log's start offset, for remote replicas its value is only updated by follower fetch
+  @volatile private[this] var _logStartOffset = Log.UnknownOffset
+
+  // The log end offset value at the time the leader received the last FetchRequest from this follower
+  // This is used to determine the lastCaughtUpTimeMs of the follower
+  @volatile private[this] var lastFetchLeaderLogEndOffset = 0L
+
+  // The time when the leader received the last FetchRequest from this follower
+  // This is used to determine the lastCaughtUpTimeMs of the follower
+  @volatile private[this] var lastFetchTimeMs = 0L
+
+  // lastCaughtUpTimeMs is the largest time t such that the offset of most recent FetchRequest from this follower >=
+  // the LEO of leader at time t. This is used to determine the lag of this follower and ISR of this partition.
+  @volatile private[this] var _lastCaughtUpTimeMs = 0L
+
+  // highWatermark is the leader's high watermark after the most recent FetchRequest from this follower. This is
+  // used to determine the maximum HW this follower knows about. See KIP-392
+  @volatile private[this] var _lastSentHighWatermark = 0L
+
+  def logStartOffset: Long = _logStartOffset
+
+  def logEndOffsetMetadata: LogOffsetMetadata = _logEndOffsetMetadata
+
+  def logEndOffset: Long = logEndOffsetMetadata.messageOffset
+
+  def lastCaughtUpTimeMs: Long = _lastCaughtUpTimeMs
+
+  def lastSentHighWatermark: Long = _lastSentHighWatermark
+
+  /*
+   * If the FetchRequest reads up to the log end offset of the leader when the current fetch request is received,
+   * set `lastCaughtUpTimeMs` to the time when the current fetch request was received.
+   *
+   * Else if the FetchRequest reads up to the log end offset of the leader when the previous fetch request was received,
+   * set `lastCaughtUpTimeMs` to the time when the previous fetch request was received.
+   *
+   * This is needed to enforce the semantics of ISR, i.e. a replica is in ISR if and only if it lags behind leader's LEO
+   * by at most `replicaLagTimeMaxMs`. These semantics allow a follower to be added to the ISR even if the offset of its
+   * fetch request is always smaller than the leader's LEO, which can happen if small produce requests are received at
+   * high frequency.
+   */
+  def updateFetchState(followerFetchOffsetMetadata: LogOffsetMetadata,
+                       followerStartOffset: Long,
+                       followerFetchTimeMs: Long,
+                       leaderEndOffset: Long,
+                       lastSentHighwatermark: Long): Unit = {
+    if (followerFetchOffsetMetadata.messageOffset >= leaderEndOffset)
+      _lastCaughtUpTimeMs = math.max(_lastCaughtUpTimeMs, followerFetchTimeMs)
+    else if (followerFetchOffsetMetadata.messageOffset >= lastFetchLeaderLogEndOffset)
+      _lastCaughtUpTimeMs = math.max(_lastCaughtUpTimeMs, lastFetchTimeMs)
+
+    _logStartOffset = followerStartOffset
+    _logEndOffsetMetadata = followerFetchOffsetMetadata
+    lastFetchLeaderLogEndOffset = leaderEndOffset
+    lastFetchTimeMs = followerFetchTimeMs
+    updateLastSentHighWatermark(lastSentHighwatermark)
+    trace(s"Updated state of replica to $this")
+  }
+
+  /**
+    * Update the high watermark of this remote replica. This is used to track what we think is the last known HW to
+    * a remote follower. Since this is recorded when we send a response, there is no way to guarantee that the follower
+    * actually receives this HW. So we consider this to be an upper bound on what the follower knows.
+    *
+    * When handling fetches, the last sent high watermark for a replica is checked to see if we should return immediately
+    * in order to propagate the HW more expeditiously. See KIP-392
+    */
+  private def updateLastSentHighWatermark(highWatermark: Long): Unit = {
+    _lastSentHighWatermark = highWatermark
+    trace(s"Updated HW of replica to $highWatermark")
+  }
+
+  def resetLastCaughtUpTime(curLeaderLogEndOffset: Long, curTimeMs: Long, lastCaughtUpTimeMs: Long): Unit = {
+    lastFetchLeaderLogEndOffset = curLeaderLogEndOffset
+    lastFetchTimeMs = curTimeMs
+    _lastCaughtUpTimeMs = lastCaughtUpTimeMs
+    trace(s"Reset state of replica to $this")
+  }
+
+  override def toString: String = {
+    val replicaString = new StringBuilder
+    replicaString.append("Replica(replicaId=" + brokerId)
+    replicaString.append(s", topic=${topicPartition.topic}")
+    replicaString.append(s", partition=${topicPartition.partition}")
+    replicaString.append(s", lastCaughtUpTimeMs=$lastCaughtUpTimeMs")
+    replicaString.append(s", logStartOffset=$logStartOffset")
+    replicaString.append(s", logEndOffset=$logEndOffset")
+    replicaString.append(s", logEndOffsetMetadata=$logEndOffsetMetadata")
+    replicaString.append(s", lastFetchLeaderLogEndOffset=$lastFetchLeaderLogEndOffset")
+    replicaString.append(s", lastFetchTimeMs=$lastFetchTimeMs")
+    replicaString.append(s", lastSentHighWatermark=$lastSentHighWatermark")
+    replicaString.append(")")
+    replicaString.toString
+  }
+
+  override def equals(that: Any): Boolean = that match {
+    case other: Replica => brokerId == other.brokerId && topicPartition == other.topicPartition
+    case _ => false
+  }
+
+  override def hashCode: Int = 31 + topicPartition.hashCode + 17 * brokerId
+}

core/src/main/scala/kafka/common/AdminCommandFailedException.scala

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+class AdminCommandFailedException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this() = this(null, null)
+}

core/src/main/scala/kafka/common/BaseEnum.scala

@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.common
+
+/*
+ * We inherit from `Product` and `Serializable` because `case` objects and classes inherit from them and if we don't
+ * do it here, the compiler will infer types that unexpectedly include `Product` and `Serializable`, see
+ * http://underscore.io/blog/posts/2015/06/04/more-on-sealed.html for more information.
+ */
+trait BaseEnum extends Product with Serializable {
+  def name: String
+}

core/src/main/scala/kafka/common/BrokerEndPointNotAvailableException.scala

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+class BrokerEndPointNotAvailableException(message: String) extends RuntimeException(message) {
+  def this() = this(null)
+}

core/src/main/scala/kafka/common/ClientIdAndBroker.scala

@@ -0,0 +1,34 @@
+package kafka.common
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Convenience case class since (clientId, brokerInfo) pairs are used to create
+ * SyncProducer Request Stats and SimpleConsumer Request and Response Stats.
+ */
+
+trait ClientIdBroker {
+}
+
+case class ClientIdAndBroker(clientId: String, brokerHost: String, brokerPort: Int) extends ClientIdBroker {
+  override def toString = "%s-%s-%d".format(clientId, brokerHost, brokerPort)
+}
+
+case class ClientIdAllBrokers(clientId: String) extends ClientIdBroker {
+  override def toString = "%s-%s".format(clientId, "AllBrokers")
+}

core/src/main/scala/kafka/common/ClientIdAndTopic.scala

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Convenience case class since (clientId, topic) pairs are used in the creation
+ * of many Stats objects.
+ */
+trait ClientIdTopic {
+}
+
+case class ClientIdAndTopic(clientId: String, topic: String) extends ClientIdTopic {
+  override def toString = "%s-%s".format(clientId, topic)
+}
+
+case class ClientIdAllTopics(clientId: String) extends ClientIdTopic {
+  override def toString = "%s-%s".format(clientId, "AllTopics")
+}
+
+

core/src/main/scala/kafka/common/Config.scala

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+import util.matching.Regex
+import kafka.utils.Logging
+import org.apache.kafka.common.errors.InvalidConfigurationException
+
+trait Config extends Logging {
+
+  def validateChars(prop: String, value: String): Unit = {
+    val legalChars = "[a-zA-Z0-9\\._\\-]"
+    val rgx = new Regex(legalChars + "*")
+
+    rgx.findFirstIn(value) match {
+      case Some(t) =>
+        if (!t.equals(value))
+          throw new InvalidConfigurationException(prop + " " + value + " is illegal, contains a character other than ASCII alphanumerics, '.', '_' and '-'")
+      case None => throw new InvalidConfigurationException(prop + " " + value + " is illegal, contains a character other than ASCII alphanumerics, '.', '_' and '-'")
+    }
+  }
+}
+
+
+
+

core/src/main/scala/kafka/common/GenerateBrokerIdException.scala

@@ -0,0 +1,27 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.common
+
+/**
+  * Thrown when there is a failure to generate a zookeeper sequenceId to use as brokerId
+  */
+class GenerateBrokerIdException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this(cause: Throwable) = this(null, cause)
+  def this() = this(null, null)
+}

core/src/main/scala/kafka/common/InconsistentBrokerIdException.scala

@@ -0,0 +1,27 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.common
+
+/**
+  * Indicates the brokerId stored in logDirs is not consistent across logDirs.
+  */
+class InconsistentBrokerIdException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this(cause: Throwable) = this(null, cause)
+  def this() = this(null, null)
+}

core/src/main/scala/kafka/common/InconsistentBrokerMetadataException.scala

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the BrokerMetadata stored in logDirs is not consistent across logDirs.
+ */
+class InconsistentBrokerMetadataException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this(cause: Throwable) = this(null, cause)
+  def this() = this(null, null)
+}

core/src/main/scala/kafka/common/InconsistentClusterIdException.scala

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the clusterId stored in logDirs is not consistent with the clusterIs stored in ZK.
+ */
+class InconsistentClusterIdException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this(cause: Throwable) = this(null, cause)
+  def this() = this(null, null)
+}

core/src/main/scala/kafka/common/IndexOffsetOverflowException.scala

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates that an attempt was made to append a message whose offset could cause the index offset to overflow.
+ */
+class IndexOffsetOverflowException(message: String, cause: Throwable) extends org.apache.kafka.common.KafkaException(message, cause) {
+  def this(message: String) = this(message, null)
+}

core/src/main/scala/kafka/common/InterBrokerSendThread.scala

@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.common
+
+import java.util.{ArrayDeque, ArrayList, Collection, Collections, HashMap, Iterator}
+import java.util.Map.Entry
+
+import kafka.utils.ShutdownableThread
+import org.apache.kafka.clients.{ClientRequest, ClientResponse, NetworkClient, RequestCompletionHandler}
+import org.apache.kafka.common.Node
+import org.apache.kafka.common.errors.AuthenticationException
+import org.apache.kafka.common.internals.FatalExitError
+import org.apache.kafka.common.requests.AbstractRequest
+import org.apache.kafka.common.utils.Time
+
+import scala.collection.JavaConverters._
+
+/**
+ *  Class for inter-broker send thread that utilize a non-blocking network client.
+ */
+abstract class InterBrokerSendThread(name: String,
+                                     networkClient: NetworkClient,
+                                     time: Time,
+                                     isInterruptible: Boolean = true)
+  extends ShutdownableThread(name, isInterruptible) {
+
+  def generateRequests(): Iterable[RequestAndCompletionHandler]
+  def requestTimeoutMs: Int
+  private val unsentRequests = new UnsentRequests
+
+  def hasUnsentRequests = unsentRequests.iterator().hasNext
+
+  override def shutdown(): Unit = {
+    initiateShutdown()
+    // wake up the thread in case it is blocked inside poll
+    networkClient.wakeup()
+    awaitShutdown()
+  }
+
+  override def doWork(): Unit = {
+    var now = time.milliseconds()
+
+    generateRequests().foreach { request =>
+      val completionHandler = request.handler
+      unsentRequests.put(request.destination,
+        networkClient.newClientRequest(request.destination.idString, request.request, now, true,
+          requestTimeoutMs, completionHandler))
+    }
+
+    try {
+      val timeout = sendRequests(now)
+      networkClient.poll(timeout, now)
+      now = time.milliseconds()
+      checkDisconnects(now)
+      failExpiredRequests(now)
+      unsentRequests.clean()
+    } catch {
+      case e: FatalExitError => throw e
+      case t: Throwable =>
+        error(s"unhandled exception caught in InterBrokerSendThread", t)
+        // rethrow any unhandled exceptions as FatalExitError so the JVM will be terminated
+        // as we will be in an unknown state with potentially some requests dropped and not
+        // being able to make progress. Known and expected Errors should have been appropriately
+        // dealt with already.
+        throw new FatalExitError()
+    }
+  }
+
+  private def sendRequests(now: Long): Long = {
+    var pollTimeout = Long.MaxValue
+    for (node <- unsentRequests.nodes.asScala) {
+      val requestIterator = unsentRequests.requestIterator(node)
+      while (requestIterator.hasNext) {
+        val request = requestIterator.next
+        if (networkClient.ready(node, now)) {
+          networkClient.send(request, now)
+          requestIterator.remove()
+        } else
+          pollTimeout = Math.min(pollTimeout, networkClient.connectionDelay(node, now))
+      }
+    }
+    pollTimeout
+  }
+
+  private def checkDisconnects(now: Long): Unit = {
+    // any disconnects affecting requests that have already been transmitted will be handled
+    // by NetworkClient, so we just need to check whether connections for any of the unsent
+    // requests have been disconnected; if they have, then we complete the corresponding future
+    // and set the disconnect flag in the ClientResponse
+    val iterator = unsentRequests.iterator()
+    while (iterator.hasNext) {
+      val entry = iterator.next
+      val (node, requests) = (entry.getKey, entry.getValue)
+      if (!requests.isEmpty && networkClient.connectionFailed(node)) {
+        iterator.remove()
+        for (request <- requests.asScala) {
+          val authenticationException = networkClient.authenticationException(node)
+          if (authenticationException != null)
+            error(s"Failed to send the following request due to authentication error: $request")
+          completeWithDisconnect(request, now, authenticationException)
+        }
+      }
+    }
+  }
+
+  private def failExpiredRequests(now: Long): Unit = {
+    // clear all expired unsent requests
+    val timedOutRequests = unsentRequests.removeAllTimedOut(now)
+    for (request <- timedOutRequests.asScala) {
+      debug(s"Failed to send the following request after ${request.requestTimeoutMs} ms: $request")
+      completeWithDisconnect(request, now, null)
+    }
+  }
+
+  def completeWithDisconnect(request: ClientRequest,
+                             now: Long,
+                             authenticationException: AuthenticationException): Unit = {
+    val handler = request.callback
+    handler.onComplete(new ClientResponse(request.makeHeader(request.requestBuilder().latestAllowedVersion()),
+      handler, request.destination, now /* createdTimeMs */ , now /* receivedTimeMs */ , true /* disconnected */ ,
+      null /* versionMismatch */ , authenticationException, null))
+  }
+
+  def wakeup(): Unit = networkClient.wakeup()
+}
+
+case class RequestAndCompletionHandler(destination: Node, request: AbstractRequest.Builder[_ <: AbstractRequest],
+                                       handler: RequestCompletionHandler)
+
+private class UnsentRequests {
+  private val unsent = new HashMap[Node, ArrayDeque[ClientRequest]]
+
+  def put(node: Node, request: ClientRequest): Unit = {
+    var requests = unsent.get(node)
+    if (requests == null) {
+      requests = new ArrayDeque[ClientRequest]
+      unsent.put(node, requests)
+    }
+    requests.add(request)
+  }
+
+  def removeAllTimedOut(now: Long): Collection[ClientRequest] = {
+    val expiredRequests = new ArrayList[ClientRequest]
+    for (requests <- unsent.values.asScala) {
+      val requestIterator = requests.iterator
+      var foundExpiredRequest = false
+      while (requestIterator.hasNext && !foundExpiredRequest) {
+        val request = requestIterator.next
+        val elapsedMs = Math.max(0, now - request.createdTimeMs)
+        if (elapsedMs > request.requestTimeoutMs) {
+          expiredRequests.add(request)
+          requestIterator.remove()
+          foundExpiredRequest = true
+        }
+      }
+    }
+    expiredRequests
+  }
+
+  def clean(): Unit = {
+    val iterator = unsent.values.iterator
+    while (iterator.hasNext) {
+      val requests = iterator.next
+      if (requests.isEmpty)
+        iterator.remove()
+    }
+  }
+
+  def iterator(): Iterator[Entry[Node, ArrayDeque[ClientRequest]]] = {
+    unsent.entrySet().iterator()
+  }
+
+  def requestIterator(node: Node): Iterator[ClientRequest] = {
+    val requests = unsent.get(node)
+    if (requests == null)
+      Collections.emptyIterator[ClientRequest]
+    else
+      requests.iterator
+  }
+
+  def nodes = unsent.keySet
+}

core/src/main/scala/kafka/common/KafkaException.scala

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package kafka.common
+
+/**
+ * Usage of this class is discouraged. Use org.apache.kafka.common.KafkaException instead.
+ *
+ * This class will be removed once kafka.security.auth classes are removed.
+ */
+class KafkaException(message: String, t: Throwable) extends RuntimeException(message, t) {
+  def this(message: String) = this(message, null)
+  def this(t: Throwable) = this("", t)
+}

core/src/main/scala/kafka/common/LogCleaningAbortedException.scala

@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Thrown when a log cleaning task is requested to be aborted.
+ */
+class LogCleaningAbortedException() extends RuntimeException() {
+}

core/src/main/scala/kafka/common/LogSegmentOffsetOverflowException.scala

@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+import kafka.log.LogSegment
+
+/**
+ * Indicates that the log segment contains one or more messages that overflow the offset (and / or time) index. This is
+ * not a typical scenario, and could only happen when brokers have log segments that were created before the patch for
+ * KAFKA-5413. With KAFKA-6264, we have the ability to split such log segments into multiple log segments such that we
+ * do not have any segments with offset overflow.
+ */
+class LogSegmentOffsetOverflowException(val segment: LogSegment, val offset: Long)
+  extends org.apache.kafka.common.KafkaException(s"Detected offset overflow at offset $offset in segment $segment") {
+}

core/src/main/scala/kafka/common/LongRef.scala

@@ -0,0 +1,61 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.common
+
+/**
+  * A mutable cell that holds a value of type `Long`. One should generally prefer using value-based programming (i.e.
+  * passing and returning `Long` values), but this class can be useful in some scenarios.
+  *
+  * Unlike `AtomicLong`, this class is not thread-safe and there are no atomicity guarantees.
+  */
+class LongRef(var value: Long) {
+
+  def addAndGet(delta: Long): Long = {
+    value += delta
+    value
+  }
+
+  def getAndAdd(delta: Long): Long = {
+    val result = value
+    value += delta
+    result
+  }
+
+  def getAndIncrement(): Long = {
+    val v = value
+    value += 1
+    v
+  }
+
+  def incrementAndGet(): Long = {
+    value += 1
+    value
+  }
+
+  def getAndDecrement(): Long = {
+    val v = value
+    value -= 1
+    v
+  }
+
+  def decrementAndGet(): Long = {
+    value -= 1
+    value
+  }
+
+}

core/src/main/scala/kafka/common/MessageFormatter.scala

@@ -0,0 +1,39 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.common
+
+import java.io.PrintStream
+import java.util.Properties
+
+import org.apache.kafka.clients.consumer.ConsumerRecord
+
+/**
+  * Typical implementations of this interface convert a `ConsumerRecord` into a type that can then be passed to
+  * a `PrintStream`.
+  *
+  * This is used by the `ConsoleConsumer`.
+  */
+trait MessageFormatter {
+
+  def init(props: Properties): Unit = {}
+
+  def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit
+
+  def close(): Unit = {}
+
+}

core/src/main/scala/kafka/common/MessageReader.scala

@@ -0,0 +1,39 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *    http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.common
+
+import java.io.InputStream
+import java.util.Properties
+
+import org.apache.kafka.clients.producer.ProducerRecord
+
+/**
+  * Typical implementations of this interface convert data from an `InputStream` received via `init` into a
+  * `ProducerRecord` instance on each invocation of `readMessage`.
+  *
+  * This is used by the `ConsoleProducer`.
+  */
+trait MessageReader {
+
+  def init(inputStream: InputStream, props: Properties): Unit = {}
+
+  def readMessage(): ProducerRecord[Array[Byte], Array[Byte]]
+
+  def close(): Unit = {}
+
+}

core/src/main/scala/kafka/common/NoEpochForPartitionException.scala

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Thrown when a get epoch request is made for partition, but no epoch exists for that partition
+ */
+class NoEpochForPartitionException(message: String) extends RuntimeException(message) {
+  def this() = this(null)
+}

core/src/main/scala/kafka/common/OffsetAndMetadata.scala

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+import java.util.Optional
+
+case class OffsetAndMetadata(offset: Long,
+                             leaderEpoch: Optional[Integer],
+                             metadata: String,
+                             commitTimestamp: Long,
+                             expireTimestamp: Option[Long]) {
+
+
+  override def toString: String  = {
+    s"OffsetAndMetadata(offset=$offset" +
+      s", leaderEpoch=$leaderEpoch" +
+      s", metadata=$metadata" +
+      s", commitTimestamp=$commitTimestamp" +
+      s", expireTimestamp=$expireTimestamp)"
+  }
+}
+
+object OffsetAndMetadata {
+  val NoMetadata: String = ""
+
+  def apply(offset: Long, metadata: String, commitTimestamp: Long): OffsetAndMetadata = {
+    OffsetAndMetadata(offset, Optional.empty(), metadata, commitTimestamp, None)
+  }
+
+  def apply(offset: Long, metadata: String, commitTimestamp: Long, expireTimestamp: Long): OffsetAndMetadata = {
+    OffsetAndMetadata(offset, Optional.empty(), metadata, commitTimestamp, Some(expireTimestamp))
+  }
+
+  def apply(offset: Long, leaderEpoch: Optional[Integer], metadata: String, commitTimestamp: Long): OffsetAndMetadata = {
+    OffsetAndMetadata(offset, leaderEpoch, metadata, commitTimestamp, None)
+  }
+}

core/src/main/scala/kafka/common/OffsetsOutOfOrderException.scala

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the follower received records with non-monotonically increasing offsets
+ */
+class OffsetsOutOfOrderException(message: String) extends RuntimeException(message) {
+}
+

core/src/main/scala/kafka/common/RecordValidationException.scala

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+import org.apache.kafka.common.errors.ApiException
+import org.apache.kafka.common.requests.ProduceResponse.RecordError
+
+import scala.collection.Seq
+
+class RecordValidationException(val invalidException: ApiException,
+                                val recordErrors: Seq[RecordError]) extends RuntimeException {
+}

core/src/main/scala/kafka/common/StateChangeFailedException.scala

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+class StateChangeFailedException(message: String, cause: Throwable) extends RuntimeException(message, cause) {
+  def this(message: String) = this(message, null)
+  def this() = this(null, null)
+}

core/src/main/scala/kafka/common/ThreadShutdownException.scala

@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * An exception that indicates a thread is being shut down normally.
+ */
+class ThreadShutdownException() extends RuntimeException {
+}

core/src/main/scala/kafka/common/TopicAlreadyMarkedForDeletionException.scala

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+class TopicAlreadyMarkedForDeletionException(message: String) extends RuntimeException(message) {
+}

core/src/main/scala/kafka/common/UnexpectedAppendOffsetException.scala

@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the follower or the future replica received records from the leader (or current
+ * replica) with first offset less than expected next offset. 
+ * @param firstOffset The first offset of the records to append
+ * @param lastOffset  The last offset of the records to append
+ */
+class UnexpectedAppendOffsetException(val message: String,
+                                      val firstOffset: Long,
+                                      val lastOffset: Long) extends RuntimeException(message) {
+}

core/src/main/scala/kafka/common/UnknownCodecException.scala

@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.common
+
+/**
+ * Indicates the client has requested a range no longer available on the server
+ */
+class UnknownCodecException(message: String) extends RuntimeException(message) {
+  def this() = this(null)
+}
+

core/src/main/scala/kafka/common/ZkNodeChangeNotificationListener.scala

@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package kafka.common
+
+import java.nio.charset.StandardCharsets.UTF_8
+import java.util.concurrent.LinkedBlockingQueue
+import java.util.concurrent.atomic.AtomicBoolean
+
+import kafka.utils.{Logging, ShutdownableThread}
+import kafka.zk.{KafkaZkClient, StateChangeHandlers}
+import kafka.zookeeper.{StateChangeHandler, ZNodeChildChangeHandler}
+import org.apache.kafka.common.utils.Time
+
+import scala.collection.Seq
+import scala.util.{Failure, Try}
+
+/**
+ * Handle the notificationMessage.
+ */
+trait NotificationHandler {
+  def processNotification(notificationMessage: Array[Byte]): Unit
+}
+
+/**
+ * A listener that subscribes to seqNodeRoot for any child changes where all children are assumed to be sequence node
+ * with seqNodePrefix. When a child is added under seqNodeRoot this class gets notified, it looks at lastExecutedChange
+ * number to avoid duplicate processing and if it finds an unprocessed child, it reads its data and calls supplied
+ * notificationHandler's processNotification() method with the child's data as argument. As part of processing these changes it also
+ * purges any children with currentTime - createTime > changeExpirationMs.
+ *
+ * @param zkClient
+ * @param seqNodeRoot
+ * @param seqNodePrefix
+ * @param notificationHandler
+ * @param changeExpirationMs
+ * @param time
+ */
+class ZkNodeChangeNotificationListener(private val zkClient: KafkaZkClient,
+                                       private val seqNodeRoot: String,
+                                       private val seqNodePrefix: String,
+                                       private val notificationHandler: NotificationHandler,
+                                       private val changeExpirationMs: Long = 15 * 60 * 1000,
+                                       private val time: Time = Time.SYSTEM) extends Logging {
+  private var lastExecutedChange = -1L
+  private val queue = new LinkedBlockingQueue[ChangeNotification]
+  private val thread = new ChangeEventProcessThread(s"$seqNodeRoot-event-process-thread")
+  private val isClosed = new AtomicBoolean(false)
+
+  def init(): Unit = {
+    zkClient.registerStateChangeHandler(ZkStateChangeHandler)
+    zkClient.registerZNodeChildChangeHandler(ChangeNotificationHandler)
+    addChangeNotification()
+    thread.start()
+  }
+
+  def close() = {
+    isClosed.set(true)
+    zkClient.unregisterStateChangeHandler(ZkStateChangeHandler.name)
+    zkClient.unregisterZNodeChildChangeHandler(ChangeNotificationHandler.path)
+    queue.clear()
+    thread.shutdown()
+  }
+
+  /**
+   * Process notifications
+   */
+  private def processNotifications(): Unit = {
+    try {
+      val notifications = zkClient.getChildren(seqNodeRoot).sorted
+      if (notifications.nonEmpty) {
+        info(s"Processing notification(s) to $seqNodeRoot")
+        val now = time.milliseconds
+        for (notification <- notifications) {
+          val changeId = changeNumber(notification)
+          if (changeId > lastExecutedChange) {
+            processNotification(notification)
+            lastExecutedChange = changeId
+          }
+        }
+        purgeObsoleteNotifications(now, notifications)
+      }
+    } catch {
+      case e: InterruptedException => if (!isClosed.get) error(s"Error while processing notification change for path = $seqNodeRoot", e)
+      case e: Exception => error(s"Error while processing notification change for path = $seqNodeRoot", e)
+    }
+  }
+
+  private def processNotification(notification: String): Unit = {
+    val changeZnode = seqNodeRoot + "/" + notification
+    val (data, _) = zkClient.getDataAndStat(changeZnode)
+    data match {
+      case Some(d) => Try(notificationHandler.processNotification(d)) match {
+        case Failure(e) => error(s"error processing change notification ${new String(d, UTF_8)} from $changeZnode", e)
+        case _ =>
+      }
+      case None => warn(s"read null data from $changeZnode")
+    }
+  }
+
+  private def addChangeNotification(): Unit = {
+    if (!isClosed.get && queue.peek() == null)
+      queue.put(new ChangeNotification)
+  }
+
+  class ChangeNotification {
+    def process(): Unit = processNotifications
+  }
+
+  /**
+   * Purges expired notifications.
+   *
+   * @param now
+   * @param notifications
+   */
+  private def purgeObsoleteNotifications(now: Long, notifications: Seq[String]): Unit = {
+    for (notification <- notifications.sorted) {
+      val notificationNode = seqNodeRoot + "/" + notification
+      val (data, stat) = zkClient.getDataAndStat(notificationNode)
+      if (data.isDefined) {
+        if (now - stat.getCtime > changeExpirationMs) {
+          debug(s"Purging change notification $notificationNode")
+          zkClient.deletePath(notificationNode)
+        }
+      }
+    }
+  }
+
+  /* get the change number from a change notification znode */
+  private def changeNumber(name: String): Long = name.substring(seqNodePrefix.length).toLong
+
+  class ChangeEventProcessThread(name: String) extends ShutdownableThread(name = name) {
+    override def doWork(): Unit = queue.take().process
+  }
+
+  object ChangeNotificationHandler extends ZNodeChildChangeHandler {
+    override val path: String = seqNodeRoot
+    override def handleChildChange(): Unit = addChangeNotification
+  }
+
+  object ZkStateChangeHandler extends  StateChangeHandler {
+    override val name: String = StateChangeHandlers.zkNodeChangeListenerHandler(seqNodeRoot)
+    override def afterInitializingSession(): Unit = addChangeNotification
+  }
+}
+

core/src/main/scala/kafka/consumer/BaseConsumerRecord.scala

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.consumer
+
+import org.apache.kafka.common.header.Headers
+import org.apache.kafka.common.header.internals.RecordHeaders
+import org.apache.kafka.common.record.{RecordBatch, TimestampType}
+
+@deprecated("This class has been deprecated and will be removed in a future release. " +
+  "Please use org.apache.kafka.clients.consumer.ConsumerRecord instead.", "0.11.0.0")
+case class BaseConsumerRecord(topic: String,
+                              partition: Int,
+                              offset: Long,
+                              timestamp: Long = RecordBatch.NO_TIMESTAMP,
+                              timestampType: TimestampType = TimestampType.NO_TIMESTAMP_TYPE,
+                              key: Array[Byte],
+                              value: Array[Byte],
+                              headers: Headers = new RecordHeaders())

core/src/main/scala/kafka/controller/ControllerChannelManager.scala

@@ -0,0 +1,605 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package kafka.controller
+
+import java.net.SocketTimeoutException
+import java.util.concurrent.{BlockingQueue, LinkedBlockingQueue, TimeUnit}
+
+import com.yammer.metrics.core.{Gauge, Timer}
+import kafka.api._
+import kafka.cluster.Broker
+import kafka.metrics.KafkaMetricsGroup
+import kafka.server.KafkaConfig
+import kafka.utils._
+import org.apache.kafka.clients._
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.metrics.Metrics
+import org.apache.kafka.common.network._
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.JaasContext
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.apache.kafka.common.utils.{LogContext, Time}
+import org.apache.kafka.common.{KafkaException, Node, Reconfigurable, TopicPartition}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.{HashMap, ListBuffer}
+import scala.collection.{Seq, Set, mutable}
+
+object ControllerChannelManager {
+  val QueueSizeMetricName = "QueueSize"
+  val RequestRateAndQueueTimeMetricName = "RequestRateAndQueueTimeMs"
+}
+
+class ControllerChannelManager(controllerContext: ControllerContext,
+                               config: KafkaConfig,
+                               time: Time,
+                               metrics: Metrics,
+                               stateChangeLogger: StateChangeLogger,
+                               threadNamePrefix: Option[String] = None) extends Logging with KafkaMetricsGroup {
+  import ControllerChannelManager._
+
+  protected val brokerStateInfo = new HashMap[Int, ControllerBrokerStateInfo]
+  private val brokerLock = new Object
+  this.logIdent = "[Channel manager on controller " + config.brokerId + "]: "
+
+  newGauge("TotalQueueSize",
+    () => brokerLock synchronized {
+      brokerStateInfo.values.iterator.map(_.messageQueue.size).sum
+    }
+  )
+
+  def startup() = {
+    controllerContext.liveOrShuttingDownBrokers.foreach(addNewBroker)
+
+    brokerLock synchronized {
+      brokerStateInfo.foreach(brokerState => startRequestSendThread(brokerState._1))
+    }
+  }
+
+  def shutdown() = {
+    brokerLock synchronized {
+      brokerStateInfo.values.toList.foreach(removeExistingBroker)
+    }
+  }
+
+  def sendRequest(brokerId: Int, request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
+                  callback: AbstractResponse => Unit = null): Unit = {
+    brokerLock synchronized {
+      val stateInfoOpt = brokerStateInfo.get(brokerId)
+      stateInfoOpt match {
+        case Some(stateInfo) =>
+          stateInfo.messageQueue.put(QueueItem(request.apiKey, request, callback, time.milliseconds()))
+        case None =>
+          warn(s"Not sending request $request to broker $brokerId, since it is offline.")
+      }
+    }
+  }
+
+  def addBroker(broker: Broker): Unit = {
+    // be careful here. Maybe the startup() API has already started the request send thread
+    brokerLock synchronized {
+      if (!brokerStateInfo.contains(broker.id)) {
+        addNewBroker(broker)
+        startRequestSendThread(broker.id)
+      }
+    }
+  }
+
+  def removeBroker(brokerId: Int): Unit = {
+    brokerLock synchronized {
+      removeExistingBroker(brokerStateInfo(brokerId))
+    }
+  }
+
+  private def addNewBroker(broker: Broker): Unit = {
+    val messageQueue = new LinkedBlockingQueue[QueueItem]
+    debug(s"Controller ${config.brokerId} trying to connect to broker ${broker.id}")
+    val controllerToBrokerListenerName = config.controlPlaneListenerName.getOrElse(config.interBrokerListenerName)
+    val controllerToBrokerSecurityProtocol = config.controlPlaneSecurityProtocol.getOrElse(config.interBrokerSecurityProtocol)
+    val brokerNode = broker.node(controllerToBrokerListenerName)
+    val logContext = new LogContext(s"[Controller id=${config.brokerId}, targetBrokerId=${brokerNode.idString}] ")
+    val (networkClient, reconfigurableChannelBuilder) = {
+      val channelBuilder = ChannelBuilders.clientChannelBuilder(
+        controllerToBrokerSecurityProtocol,
+        JaasContext.Type.SERVER,
+        config,
+        controllerToBrokerListenerName,
+        config.saslMechanismInterBrokerProtocol,
+        time,
+        config.saslInterBrokerHandshakeRequestEnable,
+        logContext
+      )
+      val reconfigurableChannelBuilder = channelBuilder match {
+        case reconfigurable: Reconfigurable =>
+          config.addReconfigurable(reconfigurable)
+          Some(reconfigurable)
+        case _ => None
+      }
+      val selector = new Selector(
+        NetworkReceive.UNLIMITED,
+        Selector.NO_IDLE_TIMEOUT_MS,
+        metrics,
+        time,
+        "controller-channel",
+        Map("broker-id" -> brokerNode.idString).asJava,
+        false,
+        channelBuilder,
+        logContext
+      )
+      val networkClient = new NetworkClient(
+        selector,
+        new ManualMetadataUpdater(Seq(brokerNode).asJava),
+        config.brokerId.toString,
+        1,
+        0,
+        0,
+        Selectable.USE_DEFAULT_BUFFER_SIZE,
+        Selectable.USE_DEFAULT_BUFFER_SIZE,
+        config.requestTimeoutMs,
+        ClientDnsLookup.DEFAULT,
+        time,
+        false,
+        new ApiVersions,
+        logContext
+      )
+      (networkClient, reconfigurableChannelBuilder)
+    }
+    val threadName = threadNamePrefix match {
+      case None => s"Controller-${config.brokerId}-to-broker-${broker.id}-send-thread"
+      case Some(name) => s"$name:Controller-${config.brokerId}-to-broker-${broker.id}-send-thread"
+    }
+
+    val requestRateAndQueueTimeMetrics = newTimer(
+      RequestRateAndQueueTimeMetricName, TimeUnit.MILLISECONDS, TimeUnit.SECONDS, brokerMetricTags(broker.id)
+    )
+
+    val requestThread = new RequestSendThread(config.brokerId, controllerContext, messageQueue, networkClient,
+      brokerNode, config, time, requestRateAndQueueTimeMetrics, stateChangeLogger, threadName)
+    requestThread.setDaemon(false)
+
+    val queueSizeGauge = newGauge(QueueSizeMetricName, () => messageQueue.size, brokerMetricTags(broker.id))
+
+    brokerStateInfo.put(broker.id, ControllerBrokerStateInfo(networkClient, brokerNode, messageQueue,
+      requestThread, queueSizeGauge, requestRateAndQueueTimeMetrics, reconfigurableChannelBuilder))
+  }
+
+  private def brokerMetricTags(brokerId: Int) = Map("broker-id" -> brokerId.toString)
+
+  private def removeExistingBroker(brokerState: ControllerBrokerStateInfo): Unit = {
+    try {
+      // Shutdown the RequestSendThread before closing the NetworkClient to avoid the concurrent use of the
+      // non-threadsafe classes as described in KAFKA-4959.
+      // The call to shutdownLatch.await() in ShutdownableThread.shutdown() serves as a synchronization barrier that
+      // hands off the NetworkClient from the RequestSendThread to the ZkEventThread.
+      brokerState.reconfigurableChannelBuilder.foreach(config.removeReconfigurable)
+      brokerState.requestSendThread.shutdown()
+      brokerState.networkClient.close()
+      brokerState.messageQueue.clear()
+      removeMetric(QueueSizeMetricName, brokerMetricTags(brokerState.brokerNode.id))
+      removeMetric(RequestRateAndQueueTimeMetricName, brokerMetricTags(brokerState.brokerNode.id))
+      brokerStateInfo.remove(brokerState.brokerNode.id)
+    } catch {
+      case e: Throwable => error("Error while removing broker by the controller", e)
+    }
+  }
+
+  protected def startRequestSendThread(brokerId: Int): Unit = {
+    val requestThread = brokerStateInfo(brokerId).requestSendThread
+    if (requestThread.getState == Thread.State.NEW)
+      requestThread.start()
+  }
+}
+
+case class QueueItem(apiKey: ApiKeys, request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
+                     callback: AbstractResponse => Unit, enqueueTimeMs: Long)
+
+class RequestSendThread(val controllerId: Int,
+                        val controllerContext: ControllerContext,
+                        val queue: BlockingQueue[QueueItem],
+                        val networkClient: NetworkClient,
+                        val brokerNode: Node,
+                        val config: KafkaConfig,
+                        val time: Time,
+                        val requestRateAndQueueTimeMetrics: Timer,
+                        val stateChangeLogger: StateChangeLogger,
+                        name: String)
+  extends ShutdownableThread(name = name) {
+
+  logIdent = s"[RequestSendThread controllerId=$controllerId] "
+
+  private val socketTimeoutMs = config.controllerSocketTimeoutMs
+
+  override def doWork(): Unit = {
+
+    def backoff(): Unit = pause(100, TimeUnit.MILLISECONDS)
+
+    val QueueItem(apiKey, requestBuilder, callback, enqueueTimeMs) = queue.take()
+    requestRateAndQueueTimeMetrics.update(time.milliseconds() - enqueueTimeMs, TimeUnit.MILLISECONDS)
+
+    var clientResponse: ClientResponse = null
+    try {
+      var isSendSuccessful = false
+      while (isRunning && !isSendSuccessful) {
+        // if a broker goes down for a long time, then at some point the controller's zookeeper listener will trigger a
+        // removeBroker which will invoke shutdown() on this thread. At that point, we will stop retrying.
+        try {
+          if (!brokerReady()) {
+            isSendSuccessful = false
+            backoff()
+          }
+          else {
+            val clientRequest = networkClient.newClientRequest(brokerNode.idString, requestBuilder,
+              time.milliseconds(), true)
+            clientResponse = NetworkClientUtils.sendAndReceive(networkClient, clientRequest, time)
+            isSendSuccessful = true
+          }
+        } catch {
+          case e: Throwable => // if the send was not successful, reconnect to broker and resend the message
+            warn(s"Controller $controllerId epoch ${controllerContext.epoch} fails to send request $requestBuilder " +
+              s"to broker $brokerNode. Reconnecting to broker.", e)
+            networkClient.close(brokerNode.idString)
+            isSendSuccessful = false
+            backoff()
+        }
+      }
+      if (clientResponse != null) {
+        val requestHeader = clientResponse.requestHeader
+        val api = requestHeader.apiKey
+        if (api != ApiKeys.LEADER_AND_ISR && api != ApiKeys.STOP_REPLICA && api != ApiKeys.UPDATE_METADATA)
+          throw new KafkaException(s"Unexpected apiKey received: $apiKey")
+
+        val response = clientResponse.responseBody
+
+        stateChangeLogger.withControllerEpoch(controllerContext.epoch).trace(s"Received response " +
+          s"${response.toString(requestHeader.apiVersion)} for request $api with correlation id " +
+          s"${requestHeader.correlationId} sent to broker $brokerNode")
+
+        if (callback != null) {
+          callback(response)
+        }
+      }
+    } catch {
+      case e: Throwable =>
+        error(s"Controller $controllerId fails to send a request to broker $brokerNode", e)
+        // If there is any socket error (eg, socket timeout), the connection is no longer usable and needs to be recreated.
+        networkClient.close(brokerNode.idString)
+    }
+  }
+
+  private def brokerReady(): Boolean = {
+    try {
+      if (!NetworkClientUtils.isReady(networkClient, brokerNode, time.milliseconds())) {
+        if (!NetworkClientUtils.awaitReady(networkClient, brokerNode, time, socketTimeoutMs))
+          throw new SocketTimeoutException(s"Failed to connect within $socketTimeoutMs ms")
+
+        info(s"Controller $controllerId connected to $brokerNode for sending state change requests")
+      }
+
+      true
+    } catch {
+      case e: Throwable =>
+        warn(s"Controller $controllerId's connection to broker $brokerNode was unsuccessful", e)
+        networkClient.close(brokerNode.idString)
+        false
+    }
+  }
+
+  override def initiateShutdown(): Boolean = {
+    if (super.initiateShutdown()) {
+      networkClient.initiateClose()
+      true
+    } else
+      false
+  }
+}
+
+class ControllerBrokerRequestBatch(config: KafkaConfig,
+                                   controllerChannelManager: ControllerChannelManager,
+                                   controllerEventManager: ControllerEventManager,
+                                   controllerContext: ControllerContext,
+                                   stateChangeLogger: StateChangeLogger)
+  extends AbstractControllerBrokerRequestBatch(config, controllerContext, stateChangeLogger) {
+
+  def sendEvent(event: ControllerEvent): Unit = {
+    controllerEventManager.put(event)
+  }
+
+  def sendRequest(brokerId: Int,
+                  request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
+                  callback: AbstractResponse => Unit = null): Unit = {
+    controllerChannelManager.sendRequest(brokerId, request, callback)
+  }
+
+}
+
+case class StopReplicaRequestInfo(replica: PartitionAndReplica, deletePartition: Boolean)
+
+abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig,
+                                                    controllerContext: ControllerContext,
+                                                    stateChangeLogger: StateChangeLogger) extends  Logging {
+  val controllerId: Int = config.brokerId
+  val leaderAndIsrRequestMap = mutable.Map.empty[Int, mutable.Map[TopicPartition, LeaderAndIsrPartitionState]]
+  val stopReplicaRequestMap = mutable.Map.empty[Int, ListBuffer[StopReplicaRequestInfo]]
+  val updateMetadataRequestBrokerSet = mutable.Set.empty[Int]
+  val updateMetadataRequestPartitionInfoMap = mutable.Map.empty[TopicPartition, UpdateMetadataPartitionState]
+
+  def sendEvent(event: ControllerEvent): Unit
+
+  def sendRequest(brokerId: Int,
+                  request: AbstractControlRequest.Builder[_ <: AbstractControlRequest],
+                  callback: AbstractResponse => Unit = null): Unit
+
+  def newBatch(): Unit = {
+    // raise error if the previous batch is not empty
+    if (leaderAndIsrRequestMap.nonEmpty)
+      throw new IllegalStateException("Controller to broker state change requests batch is not empty while creating " +
+        s"a new one. Some LeaderAndIsr state changes $leaderAndIsrRequestMap might be lost ")
+    if (stopReplicaRequestMap.nonEmpty)
+      throw new IllegalStateException("Controller to broker state change requests batch is not empty while creating a " +
+        s"new one. Some StopReplica state changes $stopReplicaRequestMap might be lost ")
+    if (updateMetadataRequestBrokerSet.nonEmpty)
+      throw new IllegalStateException("Controller to broker state change requests batch is not empty while creating a " +
+        s"new one. Some UpdateMetadata state changes to brokers $updateMetadataRequestBrokerSet with partition info " +
+        s"$updateMetadataRequestPartitionInfoMap might be lost ")
+  }
+
+  def clear(): Unit = {
+    leaderAndIsrRequestMap.clear()
+    stopReplicaRequestMap.clear()
+    updateMetadataRequestBrokerSet.clear()
+    updateMetadataRequestPartitionInfoMap.clear()
+  }
+
+  def addLeaderAndIsrRequestForBrokers(brokerIds: Seq[Int],
+                                       topicPartition: TopicPartition,
+                                       leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
+                                       replicaAssignment: ReplicaAssignment,
+                                       isNew: Boolean): Unit = {
+
+    brokerIds.filter(_ >= 0).foreach { brokerId =>
+      val result = leaderAndIsrRequestMap.getOrElseUpdate(brokerId, mutable.Map.empty)
+      val alreadyNew = result.get(topicPartition).exists(_.isNew)
+      val leaderAndIsr = leaderIsrAndControllerEpoch.leaderAndIsr
+      result.put(topicPartition, new LeaderAndIsrPartitionState()
+        .setTopicName(topicPartition.topic)
+        .setPartitionIndex(topicPartition.partition)
+        .setControllerEpoch(leaderIsrAndControllerEpoch.controllerEpoch)
+        .setLeader(leaderAndIsr.leader)
+        .setLeaderEpoch(leaderAndIsr.leaderEpoch)
+        .setIsr(leaderAndIsr.isr.map(Integer.valueOf).asJava)
+        .setZkVersion(leaderAndIsr.zkVersion)
+        .setReplicas(replicaAssignment.replicas.map(Integer.valueOf).asJava)
+        .setAddingReplicas(replicaAssignment.addingReplicas.map(Integer.valueOf).asJava)
+        .setRemovingReplicas(replicaAssignment.removingReplicas.map(Integer.valueOf).asJava)
+        .setIsNew(isNew || alreadyNew))
+    }
+
+    addUpdateMetadataRequestForBrokers(controllerContext.liveOrShuttingDownBrokerIds.toSeq, Set(topicPartition))
+  }
+
+  def addStopReplicaRequestForBrokers(brokerIds: Seq[Int],
+                                      topicPartition: TopicPartition,
+                                      deletePartition: Boolean): Unit = {
+    brokerIds.filter(_ >= 0).foreach { brokerId =>
+      val stopReplicaInfos = stopReplicaRequestMap.getOrElseUpdate(brokerId, ListBuffer.empty[StopReplicaRequestInfo])
+      stopReplicaInfos.append(StopReplicaRequestInfo(PartitionAndReplica(topicPartition, brokerId), deletePartition))
+    }
+  }
+
+  /** Send UpdateMetadataRequest to the given brokers for the given partitions and partitions that are being deleted */
+  def addUpdateMetadataRequestForBrokers(brokerIds: Seq[Int],
+                                         partitions: collection.Set[TopicPartition]): Unit = {
+
+    def updateMetadataRequestPartitionInfo(partition: TopicPartition, beingDeleted: Boolean): Unit = {
+      controllerContext.partitionLeadershipInfo.get(partition) match {
+        case Some(LeaderIsrAndControllerEpoch(leaderAndIsr, controllerEpoch)) =>
+          val replicas = controllerContext.partitionReplicaAssignment(partition)
+          val offlineReplicas = replicas.filter(!controllerContext.isReplicaOnline(_, partition))
+          val updatedLeaderAndIsr =
+            if (beingDeleted) LeaderAndIsr.duringDelete(leaderAndIsr.isr)
+            else leaderAndIsr
+
+          val partitionStateInfo = new UpdateMetadataPartitionState()
+            .setTopicName(partition.topic)
+            .setPartitionIndex(partition.partition)
+            .setControllerEpoch(controllerEpoch)
+            .setLeader(updatedLeaderAndIsr.leader)
+            .setLeaderEpoch(updatedLeaderAndIsr.leaderEpoch)
+            .setIsr(updatedLeaderAndIsr.isr.map(Integer.valueOf).asJava)
+            .setZkVersion(updatedLeaderAndIsr.zkVersion)
+            .setReplicas(replicas.map(Integer.valueOf).asJava)
+            .setOfflineReplicas(offlineReplicas.map(Integer.valueOf).asJava)
+          updateMetadataRequestPartitionInfoMap.put(partition, partitionStateInfo)
+
+        case None =>
+          info(s"Leader not yet assigned for partition $partition. Skip sending UpdateMetadataRequest.")
+      }
+    }
+
+    updateMetadataRequestBrokerSet ++= brokerIds.filter(_ >= 0)
+    partitions.foreach(partition => updateMetadataRequestPartitionInfo(partition,
+      beingDeleted = controllerContext.topicsToBeDeleted.contains(partition.topic)))
+  }
+
+  private def sendLeaderAndIsrRequest(controllerEpoch: Int, stateChangeLog: StateChangeLogger): Unit = {
+    val leaderAndIsrRequestVersion: Short =
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 4
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV0) 3
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 2
+      else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 1
+      else 0
+
+    leaderAndIsrRequestMap.filterKeys(controllerContext.liveOrShuttingDownBrokerIds.contains).foreach {
+      case (broker, leaderAndIsrPartitionStates) =>
+        if (stateChangeLog.isTraceEnabled) {
+          leaderAndIsrPartitionStates.foreach { case (topicPartition, state) =>
+            val typeOfRequest =
+              if (broker == state.leader) "become-leader"
+              else "become-follower"
+            stateChangeLog.trace(s"Sending $typeOfRequest LeaderAndIsr request $state to broker $broker for partition $topicPartition")
+          }
+        }
+        val leaderIds = leaderAndIsrPartitionStates.map(_._2.leader).toSet
+        val leaders = controllerContext.liveOrShuttingDownBrokers.filter(b => leaderIds.contains(b.id)).map {
+          _.node(config.interBrokerListenerName)
+        }
+        val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
+        val leaderAndIsrRequestBuilder = new LeaderAndIsrRequest.Builder(leaderAndIsrRequestVersion, controllerId,
+          controllerEpoch, brokerEpoch, leaderAndIsrPartitionStates.values.toBuffer.asJava, leaders.asJava)
+        sendRequest(broker, leaderAndIsrRequestBuilder, (r: AbstractResponse) => {
+          val leaderAndIsrResponse = r.asInstanceOf[LeaderAndIsrResponse]
+          sendEvent(LeaderAndIsrResponseReceived(leaderAndIsrResponse, broker))
+        })
+
+
+    }
+    leaderAndIsrRequestMap.clear()
+  }
+
+  private def sendUpdateMetadataRequests(controllerEpoch: Int, stateChangeLog: StateChangeLogger): Unit = {
+    updateMetadataRequestPartitionInfoMap.foreach { case (tp, partitionState) =>
+      stateChangeLog.trace(s"Sending UpdateMetadata request $partitionState to brokers $updateMetadataRequestBrokerSet " +
+        s"for partition $tp")
+    }
+
+    val partitionStates = updateMetadataRequestPartitionInfoMap.values.toBuffer
+    val updateMetadataRequestVersion: Short =
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 6
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 5
+      else if (config.interBrokerProtocolVersion >= KAFKA_1_0_IV0) 4
+      else if (config.interBrokerProtocolVersion >= KAFKA_0_10_2_IV0) 3
+      else if (config.interBrokerProtocolVersion >= KAFKA_0_10_0_IV1) 2
+      else if (config.interBrokerProtocolVersion >= KAFKA_0_9_0) 1
+      else 0
+
+    val liveBrokers = controllerContext.liveOrShuttingDownBrokers.iterator.map { broker =>
+      val endpoints = if (updateMetadataRequestVersion == 0) {
+        // Version 0 of UpdateMetadataRequest only supports PLAINTEXT
+        val securityProtocol = SecurityProtocol.PLAINTEXT
+        val listenerName = ListenerName.forSecurityProtocol(securityProtocol)
+        val node = broker.node(listenerName)
+        Seq(new UpdateMetadataEndpoint()
+          .setHost(node.host)
+          .setPort(node.port)
+          .setSecurityProtocol(securityProtocol.id)
+          .setListener(listenerName.value))
+      } else {
+        broker.endPoints.map { endpoint =>
+          new UpdateMetadataEndpoint()
+            .setHost(endpoint.host)
+            .setPort(endpoint.port)
+            .setSecurityProtocol(endpoint.securityProtocol.id)
+            .setListener(endpoint.listenerName.value)
+        }
+      }
+      new UpdateMetadataBroker()
+        .setId(broker.id)
+        .setEndpoints(endpoints.asJava)
+        .setRack(broker.rack.orNull)
+    }.toBuffer
+
+    updateMetadataRequestBrokerSet.intersect(controllerContext.liveOrShuttingDownBrokerIds).foreach { broker =>
+      val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(broker)
+      val updateMetadataRequestBuilder = new UpdateMetadataRequest.Builder(updateMetadataRequestVersion,
+        controllerId, controllerEpoch, brokerEpoch, partitionStates.asJava, liveBrokers.asJava)
+      sendRequest(broker, updateMetadataRequestBuilder, (r: AbstractResponse) => {
+        val updateMetadataResponse = r.asInstanceOf[UpdateMetadataResponse]
+        sendEvent(UpdateMetadataResponseReceived(updateMetadataResponse, broker))
+      })
+
+    }
+    updateMetadataRequestBrokerSet.clear()
+    updateMetadataRequestPartitionInfoMap.clear()
+  }
+
+  private def sendStopReplicaRequests(controllerEpoch: Int): Unit = {
+    val stopReplicaRequestVersion: Short =
+      if (config.interBrokerProtocolVersion >= KAFKA_2_4_IV1) 2
+      else if (config.interBrokerProtocolVersion >= KAFKA_2_2_IV0) 1
+      else 0
+
+    def stopReplicaPartitionDeleteResponseCallback(brokerId: Int)(response: AbstractResponse): Unit = {
+      val stopReplicaResponse = response.asInstanceOf[StopReplicaResponse]
+      val partitionErrorsForDeletingTopics = stopReplicaResponse.partitionErrors.asScala.iterator.filter { pe =>
+        controllerContext.isTopicDeletionInProgress(pe.topicName)
+      }.map(pe => new TopicPartition(pe.topicName, pe.partitionIndex) -> Errors.forCode(pe.errorCode)).toMap
+
+      if (partitionErrorsForDeletingTopics.nonEmpty)
+        sendEvent(TopicDeletionStopReplicaResponseReceived(brokerId, stopReplicaResponse.error, partitionErrorsForDeletingTopics))
+    }
+
+    def createStopReplicaRequest(brokerEpoch: Long, requests: Seq[StopReplicaRequestInfo], deletePartitions: Boolean): StopReplicaRequest.Builder = {
+      val partitions = requests.map(_.replica.topicPartition).asJava
+      new StopReplicaRequest.Builder(stopReplicaRequestVersion, controllerId, controllerEpoch,
+        brokerEpoch, deletePartitions, partitions)
+    }
+
+    stopReplicaRequestMap.filterKeys(controllerContext.liveOrShuttingDownBrokerIds.contains).foreach { case (brokerId, replicaInfoList) =>
+      val (stopReplicaWithDelete, stopReplicaWithoutDelete) = replicaInfoList.partition(r => r.deletePartition)
+      val brokerEpoch = controllerContext.liveBrokerIdAndEpochs(brokerId)
+
+      if (stopReplicaWithDelete.nonEmpty) {
+        debug(s"The stop replica request (delete = true) sent to broker $brokerId is ${stopReplicaWithDelete.mkString(",")}")
+        val stopReplicaRequest = createStopReplicaRequest(brokerEpoch, stopReplicaWithDelete, deletePartitions = true)
+        val callback = stopReplicaPartitionDeleteResponseCallback(brokerId) _
+        sendRequest(brokerId, stopReplicaRequest, callback)
+      }
+
+      if (stopReplicaWithoutDelete.nonEmpty) {
+        debug(s"The stop replica request (delete = false) sent to broker $brokerId is ${stopReplicaWithoutDelete.mkString(",")}")
+        val stopReplicaRequest = createStopReplicaRequest(brokerEpoch, stopReplicaWithoutDelete, deletePartitions = false)
+        sendRequest(brokerId, stopReplicaRequest)
+      }
+    }
+    stopReplicaRequestMap.clear()
+  }
+
+  def sendRequestsToBrokers(controllerEpoch: Int): Unit = {
+    try {
+      val stateChangeLog = stateChangeLogger.withControllerEpoch(controllerEpoch)
+      sendLeaderAndIsrRequest(controllerEpoch, stateChangeLog)
+      sendUpdateMetadataRequests(controllerEpoch, stateChangeLog)
+      sendStopReplicaRequests(controllerEpoch)
+    } catch {
+      case e: Throwable =>
+        if (leaderAndIsrRequestMap.nonEmpty) {
+          error("Haven't been able to send leader and isr requests, current state of " +
+            s"the map is $leaderAndIsrRequestMap. Exception message: $e")
+        }
+        if (updateMetadataRequestBrokerSet.nonEmpty) {
+          error(s"Haven't been able to send metadata update requests to brokers $updateMetadataRequestBrokerSet, " +
+            s"current state of the partition info is $updateMetadataRequestPartitionInfoMap. Exception message: $e")
+        }
+        if (stopReplicaRequestMap.nonEmpty) {
+          error("Haven't been able to send stop replica requests, current state of " +
+            s"the map is $stopReplicaRequestMap. Exception message: $e")
+        }
+        throw new IllegalStateException(e)
+    }
+  }
+}
+
+case class ControllerBrokerStateInfo(networkClient: NetworkClient,
+                                     brokerNode: Node,
+                                     messageQueue: BlockingQueue[QueueItem],
+                                     requestSendThread: RequestSendThread,
+                                     queueSizeGauge: Gauge[Int],
+                                     requestRateAndTimeMetrics: Timer,
+                                     reconfigurableChannelBuilder: Option[Reconfigurable])
+