修改从数据库下载数据的方法

mall-search/src/main/java/com/uas/search/controller/IndexController.java

@@ -71,7 +71,7 @@ public class IndexController {
     public String multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
         String message= "Downloaded: "+ indexService.multiDownloadComponent(threads, startFileIndex, endFileIndex,
-                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.NONE : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()- start) / 1000.0);
         return message;
     }
@@ -90,7 +90,7 @@ public class IndexController {
     public String multiDownloadStandardProduct(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
         String message= "Downloaded: "+ indexService.multiDownloadProduct(threads, startFileIndex, endFileIndex,
-                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.NONE : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
         return message;
     }
@@ -100,7 +100,7 @@ public class IndexController {
     public String multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
         String message= "Downloaded: "+ indexService.multiDownloadGoods(threads, startFileIndex, endFileIndex,
-                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.NONE : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
         return message;
     }
@@ -110,7 +110,7 @@ public class IndexController {
     public String multiDownloadPCB(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
         String message= "Downloaded: "+ indexService.multiDownloadPCB(threads, startFileIndex, endFileIndex,
-                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.NONE : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
         return message;
     }
@@ -120,7 +120,7 @@ public class IndexController {
     public String multiDownloadPCBGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
         String message= "Downloaded: "+ indexService.multiDownloadPCBGoods(threads, startFileIndex, endFileIndex,
-                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.NONE : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
         return message;
     }

mall-search/src/main/java/com/uas/search/service/JdbcService.java

@@ -17,6 +17,22 @@ public interface JdbcService<T> {
      */
     Long getTotalElements(String tablename);
 
+    /**
+     * 获取最大ID
+     * @param tableName
+     * @param idColumn
+     * @return
+     */
+    Long getMaxId(String tableName, String idColumn);
+
+    /**
+     * 获取最小ID
+     * @param tableName
+     * @param idColumn
+     * @return
+     */
+    Long getMinId(String tableName, String idColumn);
+
     /**
      * 获取起止id
      * @param tablename 表名

mall-search/src/main/java/com/uas/search/service/impl/IndexServiceImpl.java

@@ -98,7 +98,7 @@ public class IndexServiceImpl implements IndexService {
     /**
      * 从数据库获取数据时的分页大小
      */
-    public static final int PAGE_SIZE = 1000;
+    public static final int PAGE_SIZE = 1000 * 10;
     /**
      * 单个文件存储的最大数据数目，需是PAGE_SIZE的整数倍
      */

mall-search/src/main/java/com/uas/search/service/impl/JdbcServiceImpl.java

@@ -54,6 +54,16 @@ public class JdbcServiceImpl<T> implements JdbcService{
         return jdbcTemplate.queryForObject(String.format("select count(1) from %s", tablename), Long.class);
     }
 
+    @Override
+    public Long getMaxId(String tableName, String idColumn) {
+        return jdbcTemplate.queryForObject(String.format("select max(%s) from %s", idColumn, tableName), Long.class);
+    }
+
+    @Override
+    public Long getMinId(String tableName, String idColumn) {
+        return jdbcTemplate.queryForObject(String.format("select max(%s) from %s", idColumn, tableName), Long.class);
+    }
+
     @Override
     public List<Long> getLimitId(String tablename, String direction, String sortField, Long start, Integer size) {
         List<Long> ids = new ArrayList<>();

mall-search/src/main/java/com/uas/search/support/DownloadHelper.java

@@ -159,7 +159,7 @@ public class DownloadHelper<T> {
         this.threadSize = threadSize;
         this.downloadService = downloadService;
         this.startFileIndex = startFileIndex == null || startFileIndex < DEFAULT_START_FILE_INDEX ? DEFAULT_START_FILE_INDEX : startFileIndex;
-        this.endFileIndex = endFileIndex == null || endFileIndex < DEFAULT_START_FILE_INDEX ? DEFAULT_END_FILE_INDEX : endFileIndex;
+        this.endFileIndex = endFileIndex == null || endFileIndex > DEFAULT_END_FILE_INDEX ? DEFAULT_END_FILE_INDEX : endFileIndex;
         this.tableName = tableName;
         this.trueTableName = tableName;
         this.sortField = sortField;
@@ -179,7 +179,7 @@ public class DownloadHelper<T> {
             // 删除旧的文件
             FileUtils.deleteSubFiles(new File(SearchUtils.getDataPath(tableName)));
         }
-        // 获取下载总条数
+
         if (GOODS_TABLE_NAME.equals(tableName)) {
             tableName = TRADE_GOODS_TABLE_NAME;
             trueTableName = GOODS_TABLE_NAME;
@@ -187,14 +187,33 @@ public class DownloadHelper<T> {
             tableName = TRADE_GOODS_TABLE_NAME;
             trueTableName = PCB_GOODS_TABLE_NAME;
         }
-        getTotalElements(tableName);
+
         if (TRADE_GOODS_TABLE_NAME.equals(tableName) && GOODS_TABLE_NAME.equals(trueTableName)) {
             tableName = GOODS_TABLE_NAME;
         } else if (TRADE_GOODS_TABLE_NAME.equals(tableName) && PCB_GOODS_TABLE_NAME.equals(trueTableName)){
             tableName = PCB_GOODS_TABLE_NAME;
         }
-        logger.info(tableName + " 发现数据 " + totalElements + " 条");
-        // 线程数量不可高于下载的文件数量
+
+        if (GOODS_TABLE_NAME.equals(tableName) || PCB_GOODS_TABLE_NAME.equals(tableName)) {
+            tableName = TRADE_GOODS_TABLE_NAME;
+        }
+
+        if (TRADE_GOODS_TABLE_NAME.equals(tableName) && GOODS_TABLE_NAME.equals(trueTableName)) {
+            tableName = GOODS_TABLE_NAME;
+        } else if (TRADE_GOODS_TABLE_NAME.equals(tableName) && PCB_GOODS_TABLE_NAME.equals(trueTableName)) {
+            tableName = PCB_GOODS_TABLE_NAME;
+        }
+
+        if (GOODS_TABLE_NAME.equals(tableName) || PCB_GOODS_TABLE_NAME.equals(tableName)) {
+            tableName = TRADE_GOODS_TABLE_NAME;
+        }
+
+        // 获取下载总条数
+        totalElements = getTotalElements(tableName);
+        long minId = jdbcService.getMinId(tableName, sortField);
+        long maxId = jdbcService.getMaxId(tableName, sortField);
+        logger.info(tableName + " 发现数据 " + totalElements + " 条, id: " + minId + "~" + maxId);
+        // 线程数量高于下载的文件数量，则为文件数量
         threadSize = threadSize <= endFileIndex - startFileIndex + 1 ? threadSize : endFileIndex - startFileIndex + 1;
         for (int i = 0; i < threadSize; i++) {
             completionService.submit(getTask(i, threadSize, startFileIndex + i, endFileIndex, tableName, sortField));
@@ -228,13 +247,8 @@ public class DownloadHelper<T> {
      * 获取要下载表的数据总行数
      * @param tableName 表名
      */
-    private void getTotalElements(String tableName) {
-        totalElements = jdbcTemplate.queryForObject(String.format("select count(1) from %s", tableName), Long.class);
-//        PageParams pageParams = new PageParams();
-//        pageParams.setPage(startFileIndex);
-//        pageParams.setSize(PAGE_SIZE);
-//        PageInfo pageInfo = new PageInfo(pageParams);
-//        totalElements = dao.findAll(pageInfo).getTotalElements();
+    private Long getTotalElements(String tableName) {
+        return jdbcTemplate.queryForObject(String.format("select count(1) from %s", tableName), Long.class);
     }
 
     /**

mall-search/src/main/java/com/uas/search/support/DownloadService.java

@@ -25,9 +25,20 @@ import org.springframework.data.domain.Sort;
  *
  * @author sunyj
  * @since 2017/11/25 18:24
+ * @version 2018/7/6 修改下载数据的策略为按id范围端下载
  */
 public class DownloadService<T> {
 
+    /**
+     * 一次下载计数器，统计一次下载的总个数，包括各个线程的
+     */
+    private static long sizeCounter = 0;
+
+    /**
+     * 数据文件文件名格式
+     */
+    private static final String FILE_NAME_FORMAT = "%010d.txt";
+
     /**
      * @param id             线程 id
      * @param step           新增文件时，文件 id 的自增步长，（即线程数量）
@@ -40,9 +51,10 @@ public class DownloadService<T> {
      * @return 下载的数量
      */
     public long download(int id, int step, int startFileIndex, int endFileIndex, String tableName, String trueTableName, String sortField, JdbcService jdbcService) {
+        // 本次线程实际下载数据总数
         long size = 0L;
         Logger logger = LoggerFactory.getLogger(getClass());
-        String name = "Thread_" + id + "_" + startFileIndex + "-" + endFileIndex;
+        String name = "Thread_" + id + "_" + startFileIndex;
         File file = null;
         PrintWriter printWriter = null;
         try {
@@ -50,25 +62,21 @@ public class DownloadService<T> {
                 throw new IllegalArgumentException("SINGLE_FILE_MAX_SIZE 并非 PAGE_SIZE 的整数倍：" + SINGLE_FILE_MAX_SIZE + "/" + PAGE_SIZE);
             }
             if (startFileIndex > endFileIndex) {
-                logger.info(name + " startFileIndex 超过 " + endFileIndex);
+                logger.info(name + " startFileIndex " + startFileIndex + " 超过 endFileIndex " + endFileIndex);
                 return size;
             }
             Long startTime = System.currentTimeMillis();
             logger.info(name + " 下载 " + tableName + "...");
 
-//            Sort sort = new Sort(Sort.Direction.ASC, sortField);
             // 分页获取数据
             PageParams pageParams = new PageParams();
             pageParams.setPage((startFileIndex - 1) * SINGLE_FILE_MAX_SIZE / PAGE_SIZE + 1);
             pageParams.setSize(PAGE_SIZE);
-            PageInfo pageInfo = new PageInfo(pageParams);
-//            Page<T> pageResult = dao.findAll(pageInfo);
 
-            if (GOODS_TABLE_NAME.equals(tableName) || PCB_GOODS_TABLE_NAME.equals(tableName)) {
-                tableName = TRADE_GOODS_TABLE_NAME;
-            }
-
-            int totalFiles = (int) Math.ceil(jdbcService.getTotalElements(tableName) / (1.0 * SINGLE_FILE_MAX_SIZE));
+            Long maxId = jdbcService.getMaxId(tableName, sortField);
+            Long minId = jdbcService.getMinId(tableName, sortField);
+            // 理论上的总文件数
+            int totalFiles = (int) (maxId / SINGLE_FILE_MAX_SIZE + 1);
             if (startFileIndex > totalFiles) {
                 logger.info(name + " startFileIndex 超过 " + totalFiles);
                 return size;
@@ -76,12 +84,6 @@ public class DownloadService<T> {
             // 实际的最后一个文件
             endFileIndex = endFileIndex < totalFiles ? endFileIndex : totalFiles;
 
-            if (TRADE_GOODS_TABLE_NAME.equals(tableName) && GOODS_TABLE_NAME.equals(trueTableName)) {
-                tableName = GOODS_TABLE_NAME;
-            } else if (TRADE_GOODS_TABLE_NAME.equals(tableName) && PCB_GOODS_TABLE_NAME.equals(trueTableName)) {
-                tableName = PCB_GOODS_TABLE_NAME;
-            }
-
             File dataDir = new File(SearchUtils.getDataPath(tableName));
             if (!dataDir.exists()) {
                 if (!dataDir.mkdirs()) {
@@ -89,58 +91,46 @@ public class DownloadService<T> {
                 }
             }
             int fileIndex = startFileIndex;
-            String fileName = String.format("%010d", fileIndex) + ".txt";
-            file = new File(dataDir, fileName);
-            printWriter = new PrintWriter(file);
+            String fileName = String.format(FILE_NAME_FORMAT, fileIndex);
+            // 当前文件已获取id区间数
             int count = 0;
-            if (GOODS_TABLE_NAME.equals(tableName) || PCB_GOODS_TABLE_NAME.equals(tableName)) {
-                tableName = TRADE_GOODS_TABLE_NAME;
-            }
+            // 当前文件已获取实际数据条数
+            int fileDataSize = 0;
             while (endFileIndex >= fileIndex) {
-                List<Long> ids = jdbcService.getLimitId(tableName, Sort.Direction.ASC.name(), sortField, (long) ((fileIndex - 1) * PAGE_SIZE), PAGE_SIZE);
-                logger.info("下载id:" + ids.get(0) + "~" + ids.get(1));
-                if (pageParams.getPage() > endFileIndex && ids.size() > 2) {
-                    break;
-                }
+                Long startId = (long) ((pageParams.getPage() - 1) * pageParams.getSize() + 1);
+                Long endId = (long) (pageParams.getPage() * pageParams.getSize());
                 // 获取数据
-                List<T> data = jdbcService.getData(tableName, ids.get(0), ids.get(1), sortField, Sort.Direction.ASC.name());
+                List<T> content = jdbcService.getData(tableName, startId, endId, sortField, Sort.Direction.ASC.name());
                 // 数据以 JSON 格式写入文件
-                List<T> content = data;
-                for (T element : content) {
-                    println(printWriter, element);
+                if (!CollectionUtils.isEmpty(content)) {
+                    // 有数据，并且还没有创建新文件时，进行创建新文件
+                    if (fileDataSize == 0) {
+                        close(printWriter);
+                        fileName = String.format(FILE_NAME_FORMAT, fileIndex);
+                        file = new File(dataDir, fileName);
+                        printWriter = new PrintWriter(file);
+                    }
+
+                    for (T element : content) {
+                        println(printWriter, element);
+                    }
+
+                    size += content.size();
+                    fileDataSize += content.size();
                 }
-                count += content.size();
-                size += content.size();
-                logger.info(name + " " + fileName + " - Downloaded..................." + size);
+                count += PAGE_SIZE;
+                logger.info("{} {} download id:{}~{}, current data size: {}, total data size: {}", name, fileName, startId, endId, content.size(), size);
+                pageParams.setPage(pageParams.getPage() + 1);
 
-                // 一个文件的函数达到指定值，写入新文件
+                // 一个文件的获取id数达到指定值，写入新文件
                 if (count == SINGLE_FILE_MAX_SIZE) {
                     count = 0;
-                    close(printWriter);
+                    fileDataSize = 0;
                     fileIndex += step;
-                    pageParams.setPage(pageParams.getPage() + step * SINGLE_FILE_MAX_SIZE / PAGE_SIZE);
-                    // 不创建多余文件
-                    if (endFileIndex >= fileIndex) {
-                        fileName = String.format("%010d", fileIndex) + ".txt";
-                        file = new File(dataDir, fileName);
-                        printWriter = new PrintWriter(file);
-                    }
-                } else {
-                    pageParams.setPage(pageParams.getPage() + 1);
-                }
-//                pageInfo = new PageInfo(pageParams, sort);
-//                pageResult = dao.findAll(pageInfo);
-                // 已经下载到最后一页
-                if (CollectionUtils.isEmpty(data)) {
-                    if (TRADE_GOODS_TABLE_NAME.equals(tableName) && GOODS_TABLE_NAME.equals(trueTableName)) {
-                        tableName = GOODS_TABLE_NAME;
-                    } else if (TRADE_GOODS_TABLE_NAME.equals(tableName) && PCB_GOODS_TABLE_NAME.equals(trueTableName)) {
-                        tableName = PCB_GOODS_TABLE_NAME;
-                    }
-                    break;
+                    pageParams.setPage(pageParams.getPage() + (step -1) * SINGLE_FILE_MAX_SIZE / PAGE_SIZE);
                 }
             }
-            logger.info(String.format("%s 下载完成，耗时%.2fs\n ", name, (new Date().getTime() - startTime) / 1000.0));
+            logger.info(String.format("%s 下载完成，耗时%.2fs\n ", name, (System.currentTimeMillis() - startTime) / 1000.0));
         } catch (Throwable e) {
             close(printWriter);
             // 出现错误时，删除最近出错的文件