Merge pull request #555 from superspeedone/dev

Dev

km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/TopicStateManager.java

@@ -1,5 +1,6 @@
 package com.xiaojukeji.know.streaming.km.biz.topic;
 
+import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
 import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicRecordDTO;
 import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
 import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicBrokersPartitionsSummaryVO;

km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/impl/TopicStateManagerImpl.java

@@ -22,25 +22,26 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.partition.TopicPart
 import com.xiaojukeji.know.streaming.km.common.constant.Constant;
 import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
 import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
-import com.xiaojukeji.know.streaming.km.common.converter.PartitionConverter;
 import com.xiaojukeji.know.streaming.km.common.converter.TopicVOConverter;
+import com.xiaojukeji.know.streaming.km.common.enums.GroupOffsetResetEnum;
+import com.xiaojukeji.know.streaming.km.common.enums.SortTypeEnum;
 import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
 import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
 import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
+import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
 import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
 import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
 import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
 import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionMetricService;
-import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
 import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
+import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
 import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
 import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
 import com.xiaojukeji.know.streaming.km.core.service.version.metrics.TopicMetricVersionItems;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kafka.clients.admin.OffsetSpec;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.*;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.config.TopicConfig;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -160,8 +161,31 @@ public class TopicStateManagerImpl implements TopicStateManager {
             }
             maxMessage = Math.min(maxMessage, dto.getMaxRecords());
             kafkaConsumer.assign(partitionList);
+
+            Map<TopicPartition, OffsetAndTimestamp> partitionOffsetAndTimestampMap = new HashMap<>();
+            // 获取指定时间每个分区的offset（按指定开始时间查询消息时）
+            if (GroupOffsetResetEnum.PRECISE_TIMESTAMP.getResetType() == dto.getFilterOffsetReset()) {
+                Map<TopicPartition, Long> timestampsToSearch = new HashMap<>();
+                partitionList.forEach(topicPartition -> {
+                    timestampsToSearch.put(topicPartition, dto.getStartTimestampUnitMs());
+                });
+                partitionOffsetAndTimestampMap = kafkaConsumer.offsetsForTimes(timestampsToSearch);
+            }
+
             for (TopicPartition partition : partitionList) {
-                kafkaConsumer.seek(partition, Math.max(beginOffsetsMapResult.getData().get(partition), endOffsetsMapResult.getData().get(partition) - dto.getMaxRecords()));
+                if (GroupOffsetResetEnum.EARLIEST.getResetType() == dto.getFilterOffsetReset()) {
+                    // 重置到最旧
+                    kafkaConsumer.seek(partition, beginOffsetsMapResult.getData().get(partition));
+                } else if (GroupOffsetResetEnum.PRECISE_TIMESTAMP.getResetType() == dto.getFilterOffsetReset()) {
+                    // 重置到指定时间
+                    kafkaConsumer.seek(partition, partitionOffsetAndTimestampMap.get(partition).offset());
+                } else if (GroupOffsetResetEnum.PRECISE_OFFSET.getResetType() == dto.getFilterOffsetReset()) {
+                    // 重置到指定位置
+
+                } else {
+                    // 默认，重置到最新
+                    kafkaConsumer.seek(partition, Math.max(beginOffsetsMapResult.getData().get(partition), endOffsetsMapResult.getData().get(partition) - dto.getMaxRecords()));
+                }
             }
 
             // 这里需要减去 KafkaConstant.POLL_ONCE_TIMEOUT_UNIT_MS 是因为poll一次需要耗时，如果这里不减去，则可能会导致poll之后，超过要求的时间
@@ -185,6 +209,15 @@ public class TopicStateManagerImpl implements TopicStateManager {
                     }
             }
 
+            // 排序
+            if (ObjectUtils.isNotEmpty(voList)) {
+                // 默认按时间倒序排序
+                if (StringUtils.isBlank(dto.getSortType())) {
+                    dto.setSortType(SortTypeEnum.DESC.getSortType());
+                }
+                PaginationUtil.pageBySort(voList, dto.getSortField(), dto.getSortType());
+            }
+
             return Result.buildSuc(voList.subList(0, Math.min(dto.getMaxRecords(), voList.size())));
         } catch (Exception e) {
             log.error("method=getTopicMessages||clusterPhyId={}||topicName={}||param={}||errMsg=exception", clusterPhyId, topicName, dto, e);

km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/topic/TopicRecordDTO.java

@@ -2,6 +2,7 @@ package com.xiaojukeji.know.streaming.km.common.bean.dto.topic;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO;
+import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
@@ -15,7 +16,7 @@ import javax.validation.constraints.NotNull;
 @Data
 @JsonIgnoreProperties(ignoreUnknown = true)
 @ApiModel(description = "Topic记录")
-public class TopicRecordDTO extends BaseDTO {
+public class TopicRecordDTO extends PaginationSortDTO {
     @NotNull(message = "truncate不允许为空")
     @ApiModelProperty(value = "是否截断", example = "true")
     private Boolean truncate;
@@ -34,4 +35,14 @@ public class TopicRecordDTO extends BaseDTO {
 
     @ApiModelProperty(value = "预览超时时间", example = "10000")
     private Long pullTimeoutUnitMs = 8000L;
+
+    @ApiModelProperty(value = "offset", example = "")
+    private Integer filterOffsetReset = 0;
+
+    @ApiModelProperty(value = "开始日期时间戳", example = "")
+    private Long startTimestampUnitMs;
+
+    @ApiModelProperty(value = "结束日期时间戳", example = "")
+    private Long utilTimestampUnitMs;
+
 }

km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java

@@ -64,4 +64,5 @@ public class Constant {
     public static final Float COLLECT_METRICS_ERROR_COST_TIME = -1.0F;
 
     public static final Integer DEFAULT_RETRY_TIME = 3;
+
 }

km-console/packages/layout-clusters-fe/src/pages/TopicDetail/Messages.tsx

@@ -10,6 +10,7 @@ const defaultParams: any = {
   maxRecords: 100,
   pullTimeoutUnitMs: 5000,
   // filterPartitionId: 1,
+  filterOffsetReset: 0
 };
 const defaultpaPagination = {
   current: 1,
@@ -29,12 +30,20 @@ const TopicMessages = (props: any) => {
   const [pagination, setPagination] = useState<any>(defaultpaPagination);
   const [form] = Form.useForm();
 
+  // 获取消息开始位置
+  const offsetResetList = [
+    { 'label': 'latest', value: 0 },
+    { 'label': 'earliest', value: 1 }
+  ];
+
   // 默认排序
   const defaultSorter = {
     sortField: 'timestampUnitMs',
     sortType: 'desc',
   };
 
+  const [sorter, setSorter] = useState<any>(defaultSorter);
+
   // 请求接口获取数据
   const genData = async () => {
     if (urlParams?.clusterId === undefined || hashData?.topicName === undefined) return;
@@ -49,7 +58,7 @@ const TopicMessages = (props: any) => {
       });
       setPartitionIdList(newPartitionIdList || []);
     });
-    request(Api.getTopicMessagesList(hashData?.topicName, urlParams?.clusterId), { data: { ...params, ...defaultSorter }, method: 'POST' })
+    request(Api.getTopicMessagesList(hashData?.topicName, urlParams?.clusterId), { data: { ...params, ...sorter }, method: 'POST' })
       .then((res: any) => {
         // setPagination({
         //   current: res.pagination?.pageNo,
@@ -87,8 +96,15 @@ const TopicMessages = (props: any) => {
     history.push(`/cluster/${urlParams?.clusterId}/testing/consumer`);
   };
 
-  const onTableChange = (pagination: any, filters: any, sorter: any) => {
+  const onTableChange = (pagination: any, filters: any, sorter: any, extra: any) => {
     setPagination(pagination);
+    // 只有排序事件时，触发重新请求后端数据
+    if(extra.action === 'sort') {
+      setSorter({
+        sortField: sorter.field || '',
+        sortType: sorter.order ? sorter.order.substring(0, sorter.order.indexOf('end')) : ''
+      });
+    }
     // const asc = sorter?.order && sorter?.order === 'ascend' ? true : false;
     // const sortColumn = sorter.field && toLine(sorter.field);
     // genData({ pageNo: pagination.current, pageSize: pagination.pageSize, filters, asc, sortColumn, queryTerm: searchResult, ...allParams });
@@ -96,7 +112,7 @@ const TopicMessages = (props: any) => {
 
   useEffect(() => {
     props.positionType === 'Messages' && genData();
-  }, [props, params]);
+  }, [props, params, sorter]);
 
   return (
     <>
@@ -119,6 +135,15 @@ const TopicMessages = (props: any) => {
         </div>
         <div className="messages-query">
           <Form form={form} layout="inline" onFinish={onFinish}>
+            <Form.Item name="filterOffsetReset">
+              <Select
+                  options={offsetResetList}
+                  size="small"
+                  style={{ width: '120px' }}
+                  className={'detail-table-select'}
+                  placeholder="请选择offset"
+              />
+            </Form.Item>
             <Form.Item name="filterPartitionId">
               <Select
                 options={partitionIdList}
@@ -158,7 +183,7 @@ const TopicMessages = (props: any) => {
         showQueryForm={false}
         tableProps={{
           showHeader: false,
-          rowKey: 'path',
+          rowKey: 'offset',
           loading: loading,
           columns: getTopicMessagesColmns(),
           dataSource: data,
@@ -169,6 +194,7 @@ const TopicMessages = (props: any) => {
             bordered: false,
             onChange: onTableChange,
             scroll: { x: 'max-content' },
+            sortDirections: ['descend', 'ascend', 'default']
           },
         }}
       />

km-console/packages/layout-clusters-fe/src/pages/TopicDetail/config.tsx

@@ -85,7 +85,8 @@ export const getTopicMessagesColmns = () => {
       title: 'Timestamp',
       dataIndex: 'timestampUnitMs',
       key: 'timestampUnitMs',
-      render: (t: number) => (t ? moment(t).format(timeFormat) : '-'),
+      sorter: true,
+      render: (t: number) => (t ? moment(t).format(timeFormat) + '.' + moment(t).millisecond() : '-'),
     },
     {
       title: 'Key',

km-rest/src/main/java/com/xiaojukeji/know/streaming/km/rest/api/v3/topic/TopicStateController.java

@@ -3,6 +3,7 @@ package com.xiaojukeji.know.streaming.km.rest.api.v3.topic;
 import com.xiaojukeji.know.streaming.km.biz.topic.TopicStateManager;
 import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
 import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
+import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
 import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicRecordDTO;
 import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics;
 import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;