修改索引下载方式

mall-search/pom.xml

@@ -59,6 +59,11 @@
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>
 		</dependency>
+		<!--jdbcTemplate-->
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-jdbc</artifactId>
+		</dependency>
 		<!-- druid -->
 		<dependency>
 			<groupId>com.alibaba</groupId>

mall-search/src/main/java/com/uas/search/controller/IndexController.java

@@ -72,7 +72,7 @@ public class IndexController {
         long start = System.currentTimeMillis();
         String message= "Downloaded: "+ indexService.multiDownloadComponent(threads, startFileIndex, endFileIndex,
                 StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
-        message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
+        message += String.format(", Time: %.2fs", (System.currentTimeMillis()- start) / 1000.0);
         return message;
     }
 

mall-search/src/main/java/com/uas/search/service/JdbcService.java

@@ -0,0 +1,31 @@
+package com.uas.search.service;
+
+import java.util.List;
+import org.springframework.data.domain.Sort;
+
+/**
+ * 利用jdbcTemplate进行的主要查询
+ * Created by wangyc on 2018/6/11.
+ *
+ * @version 2018/6/11 15:47 wangyc
+ */
+public interface JdbcService {
+
+    /**
+     * 获取该表全部数据
+     * @param tablename 表名
+     * @return
+     */
+    Long getTotalElements(String tablename);
+
+    /**
+     * 获取起止id
+     * @param tablename 表名
+     * @param direction 排序方向
+     * @param sortField 排序字段
+     * @param start 起始位置
+     * @param size 间隔
+     * @return
+     */
+    List<Long> getLimitId(String tablename, String direction, String sortField, Long start, Integer size);
+}

mall-search/src/main/java/com/uas/search/service/impl/IndexServiceImpl.java

@@ -750,6 +750,11 @@ public class IndexServiceImpl implements IndexService {
 
         if (tableName.equals(COMPONENT_TABLE_NAME)) {
             DownloadHelper<Component> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id", componentDao, new DownloadService<Component>(), validateResult);
+            try {
+                Component.class.getField("id").getAnnotation();
+            } catch (NoSuchFieldException e) {
+                e.printStackTrace();
+            }
             return downloadHelper.getResult();
 
         } else if (tableName.equals(GOODS_TABLE_NAME)) {

mall-search/src/main/java/com/uas/search/service/impl/JdbcServiceImpl.java

@@ -0,0 +1,33 @@
+package com.uas.search.service.impl;
+
+import com.uas.search.service.JdbcService;
+import java.util.ArrayList;
+import java.util.List;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+/**
+ * Created by wangyc on 2018/6/11.
+ *
+ * @version 2018/6/11 15:47 wangyc
+ */
+public class JdbcServiceImpl implements JdbcService {
+
+    @Autowired
+    private JdbcTemplate jdbcTemplate;
+
+    @Override
+    public Long getTotalElements(String tablename) {
+        return jdbcTemplate.queryForObject(String.format("select count(1) from %s", tablename), Long.class);
+    }
+
+    @Override
+    public List<Long> getLimitId(String tablename, String direction, String sortField, Long start, Integer size) {
+        List<Long> ids = new ArrayList<>();
+        ids.add(jdbcTemplate.queryForObject(String.format("select %s from %s order by %s limit %s,1", sortField,
+            tablename, sortField, start), Long.class));
+        ids.add(jdbcTemplate.queryForObject(String.format("select %s from %s order by %s limit %s,1", sortField,
+            tablename, sortField, start + size - 1), Long.class));
+        return ids;
+    }
+}

mall-search/src/main/java/com/uas/search/support/DownloadHelper.java

@@ -8,12 +8,14 @@ import com.uas.search.util.FileUtils;
 import com.uas.search.util.SearchUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.jpa.repository.JpaRepository;
 
 import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.*;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import static com.uas.search.service.impl.IndexServiceImpl.PAGE_SIZE;
 import static com.uas.search.service.impl.IndexServiceImpl.SINGLE_FILE_MAX_SIZE;
@@ -110,6 +112,10 @@ public class DownloadHelper<T> {
 
     private Logger logger = LoggerFactory.getLogger(getClass());
 
+    @Autowired
+    private JdbcTemplate jdbcTemplate;
+
+
     /**
      * @param threadSize      线程数量
      * @param startFileIndex  开始的文件
@@ -161,7 +167,7 @@ public class DownloadHelper<T> {
             // 删除旧的文件
             FileUtils.deleteSubFiles(new File(SearchUtils.getDataPath(tableName)));
         }
-        getTotalElements();
+        getTotalElements(tableName);
         logger.info(tableName + " 发现数据 " + totalElements + " 条");
         // 线程数量不可高于下载的文件数量
         threadSize = threadSize <= endFileIndex - startFileIndex + 1 ? threadSize : endFileIndex - startFileIndex + 1;
@@ -195,13 +201,15 @@ public class DownloadHelper<T> {
 
     /**
      * 获取要下载表的数据总行数
+     * @param tableName 表名
      */
-    private void getTotalElements() {
-        PageParams pageParams = new PageParams();
-        pageParams.setPage(startFileIndex);
-        pageParams.setSize(PAGE_SIZE);
-        PageInfo pageInfo = new PageInfo(pageParams);
-        totalElements = dao.findAll(pageInfo).getTotalElements();
+    private void getTotalElements(String tableName) {
+        totalElements = jdbcTemplate.queryForObject("select count(1) from " + tableName, Long.class);
+//        PageParams pageParams = new PageParams();
+//        pageParams.setPage(startFileIndex);
+//        pageParams.setSize(PAGE_SIZE);
+//        PageInfo pageInfo = new PageInfo(pageParams);
+//        totalElements = dao.findAll(pageInfo).getTotalElements();
     }
 
     /**

mall-search/src/main/java/com/uas/search/support/DownloadService.java

@@ -3,11 +3,13 @@ package com.uas.search.support;
 import com.alibaba.fastjson.JSONObject;
 import com.uas.search.constant.model.PageInfo;
 import com.uas.search.constant.model.PageParams;
+import com.uas.search.service.JdbcService;
 import com.uas.search.service.impl.IndexServiceImpl;
 import com.uas.search.util.CollectionUtils;
 import com.uas.search.util.SearchUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.jpa.repository.JpaRepository;
@@ -16,6 +18,7 @@ import java.io.File;
 import java.io.PrintWriter;
 import java.util.Date;
 import java.util.List;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import static com.uas.search.service.impl.IndexServiceImpl.PAGE_SIZE;
 import static com.uas.search.service.impl.IndexServiceImpl.SINGLE_FILE_MAX_SIZE;
@@ -28,6 +31,9 @@ import static com.uas.search.service.impl.IndexServiceImpl.SINGLE_FILE_MAX_SIZE;
  */
 public class DownloadService<T> {
 
+    @Autowired
+    private JdbcService jdbcService;
+
     /**
      * @param id             线程 id
      * @param step           新增文件时，文件 id 的自增步长，（即线程数量）
@@ -52,18 +58,21 @@ public class DownloadService<T> {
                 logger.info(name + " startFileIndex 超过 " + endFileIndex);
                 return size;
             }
-            Long startTime = new Date().getTime();
+            Long startTime = System.currentTimeMillis();
             logger.info(name + " 下载 " + tableName + "...");
 
             Sort sort = new Sort(Sort.Direction.ASC, sortField);
             // 分页获取数据
-            PageParams pageParams = new PageParams();
-            pageParams.setPage((startFileIndex - 1) * SINGLE_FILE_MAX_SIZE / PAGE_SIZE + 1);
-            pageParams.setSize(IndexServiceImpl.PAGE_SIZE);
-            PageInfo pageInfo = new PageInfo(pageParams, sort);
+//            PageParams pageParams = new PageParams();
+//            pageParams.setPage((startFileIndex - 1) * SINGLE_FILE_MAX_SIZE / PAGE_SIZE + 1);
+//            pageParams.setSize(PAGE_SIZE);
+//            PageInfo pageInfo = new PageInfo(pageParams, sort);
             Page<T> pageResult = dao.findAll(pageInfo);
 
-            int totalFiles = (int) Math.ceil(pageResult.getTotalElements() / (1.0 * SINGLE_FILE_MAX_SIZE));
+            jdbcService.getLimitId(tableName, Sort.Direction.ASC.name(), sortField, (long) (startFileIndex - 1) * SINGLE_FILE_MAX_SIZE, PAGE_SIZE);
+
+
+            int totalFiles = (int) Math.ceil(jdbcService.getTotalElements(tableName) / (1.0 * SINGLE_FILE_MAX_SIZE));
             if (startFileIndex > totalFiles) {
                 logger.info(name + " startFileIndex 超过 " + totalFiles);
                 return size;