specify ValidateResult

src/main/java/com/uas/search/controller/IndexController.java

@@ -7,6 +7,7 @@ import com.uas.search.jms.LuceneMessage;
 import com.uas.search.jms.LuceneMessageDao;
 import com.uas.search.jms.LuceneMessageService;
 import com.uas.search.service.IndexService;
+import com.uas.search.support.DownloadHelper;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.util.StringUtils;
@@ -66,18 +67,20 @@ public class IndexController {
 
     @RequestMapping("/multiDownloadComponent")
     @ResponseBody
-    public String multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex, HttpServletRequest request) {
+    public String multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
-        String message= "Downloaded: "+ indexService.multiDownloadComponent(threads, startFileIndex, endFileIndex);
+        String message= "Downloaded: "+ indexService.multiDownloadComponent(threads, startFileIndex, endFileIndex,
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
         return message;
     }
 
     @RequestMapping("/multiDownloadGoods")
     @ResponseBody
-    public String multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, HttpServletRequest request) {
+    public String multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, String validateResult, HttpServletRequest request) {
         long start = System.currentTimeMillis();
-        String message= "Downloaded: "+ indexService.multiDownloadGoods(threads, startFileIndex, endFileIndex);
+        String message= "Downloaded: "+ indexService.multiDownloadGoods(threads, startFileIndex, endFileIndex,
+                StringUtils.isEmpty(validateResult) ? DownloadHelper.ValidateResult.CURRENT : DownloadHelper.ValidateResult.valueOf(validateResult.toUpperCase()));
         message += String.format(", Time: %.2fs", (System.currentTimeMillis()-start)/1000.0);
         return message;
     }

src/main/java/com/uas/search/service/IndexService.java

@@ -2,6 +2,7 @@ package com.uas.search.service;
 
 import com.uas.search.annotation.NotEmpty;
 import com.uas.search.model.ParsedQueueMessage;
+import com.uas.search.support.DownloadHelper;
 
 import java.util.List;
 import java.util.Map;
@@ -29,9 +30,10 @@ public interface IndexService {
      * @param threads        线程数量，默认为 1
      * @param startFileIndex 开始的文件，默认为 1
      * @param endFileIndex   结束的文件，默认为 1024 * 1024 * 1024
+     * @param validateResult 下载完成后，是否对结果进行校验
      * @return 下载的数据条数
      */
-    public long multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex);
+    public long multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex, DownloadHelper.ValidateResult validateResult);
 
     /**
      * 多线程下载批次的数据至本地文件中，以供建索引用
@@ -39,9 +41,10 @@ public interface IndexService {
      * @param threads        线程数量，默认为 1
      * @param startFileIndex 开始的文件，默认为 1
      * @param endFileIndex   结束的文件，默认为 1024 * 1024 * 1024
+     * @param validateResult 下载完成后，是否对结果进行校验
      * @return 下载的数据条数
      */
-    public long multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex);
+    public long multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, DownloadHelper.ValidateResult validateResult);
 
     /**
      * 将新对象添加在lucene索引中

src/main/java/com/uas/search/service/impl/IndexServiceImpl.java

@@ -415,13 +415,13 @@ public class IndexServiceImpl implements IndexService {
 	}
 
     @Override
-    public long multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex) {
-        return multiDownloadData(COMPONENT_TABLE_NAME,threads,startFileIndex,endFileIndex);
+    public long multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex, DownloadHelper.ValidateResult validateResult) {
+        return multiDownloadData(COMPONENT_TABLE_NAME,threads,startFileIndex,endFileIndex, validateResult);
     }
 
     @Override
-    public long multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex) {
-        return multiDownloadData(GOODS_TABLE_NAME,threads,startFileIndex,endFileIndex);
+    public long multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, DownloadHelper.ValidateResult validateResult) {
+        return multiDownloadData(GOODS_TABLE_NAME,threads,startFileIndex,endFileIndex, validateResult);
     }
 
     /**
@@ -431,16 +431,17 @@ public class IndexServiceImpl implements IndexService {
      * @param threads        线程数量，默认为 1
      * @param startFileIndex 开始的文件，默认为 1
      * @param endFileIndex   结束的文件，默认为 1024 * 1024 * 1024
+     * @param validateResult 下载完成后，是否对结果进行校验
      * @return 下载的数据条数
      */
-    private long multiDownloadData(String tableName, Integer threads, Integer startFileIndex, Integer endFileIndex) {
+    private long multiDownloadData(String tableName, Integer threads, Integer startFileIndex, Integer endFileIndex, DownloadHelper.ValidateResult validateResult) {
         threads = threads == null || threads < 1 ? 1 : threads;
         if (threads > druidDBConfiguration.getMaxActive()) {
             throw new IllegalArgumentException("线程数量不可超过 " + druidDBConfiguration.getMaxActive());
         }
 
         if (tableName.equals(COMPONENT_TABLE_NAME)) {
-            DownloadHelper<Component> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id",componentDao, new DownloadService<Component>());
+            DownloadHelper<Component> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id",componentDao, new DownloadService<Component>(), validateResult);
             return downloadHelper.getResult();
 
         } else if (tableName.equals(GOODS_TABLE_NAME)) {
@@ -450,7 +451,7 @@ public class IndexServiceImpl implements IndexService {
                     Goods goods = goodsDao.findByTradeGoods(element);
                     printWriter.println(JSONObject.toJSONString(goods));
                 }
-            });
+            }, validateResult);
             return downloadHelper.getResult();
         } else {
             throw new IllegalArgumentException("多线程下载不支持该表：" + tableName);

src/main/java/com/uas/search/support/DownloadHelper.java

@@ -17,6 +17,7 @@ import java.util.concurrent.*;
 
 import static com.uas.search.service.impl.IndexServiceImpl.PAGE_SIZE;
 import static com.uas.search.service.impl.IndexServiceImpl.SINGLE_FILE_MAX_SIZE;
+import static com.uas.search.support.DownloadHelper.ValidateResult.*;
 
 
 /**
@@ -85,7 +86,7 @@ public class DownloadHelper<T> {
     /**
      * 下载完成后，是否对结果进行校验
      */
-    private boolean validateResult;
+    private ValidateResult validateResult;
 
     /**
      * 线程管理
@@ -119,7 +120,7 @@ public class DownloadHelper<T> {
      * @param downloadService 下载的实现
      */
     public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, JpaRepository<T, Long> dao, DownloadService<T> downloadService) {
-        this(threadSize, startFileIndex, endFileIndex, tableName, sortField, dao, downloadService, true);
+        this(threadSize, startFileIndex, endFileIndex, tableName, sortField, dao, downloadService, CURRENT);
     }
 
     /**
@@ -132,7 +133,7 @@ public class DownloadHelper<T> {
      * @param downloadService 下载的实现
      * @param validateResult  下载完成后，是否对结果进行校验
      */
-    public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, JpaRepository<T, Long> dao, DownloadService<T> downloadService, boolean validateResult) {
+    public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, JpaRepository<T, Long> dao, DownloadService<T> downloadService, ValidateResult validateResult) {
         if (threadSize == null || threadSize < MIN_THREAD_SIZE || threadSize > MAX_THREAD_SIZE) {
             throw new IllegalArgumentException("threadSize is between " + MIN_THREAD_SIZE + " and " + MAX_THREAD_SIZE);
         }
@@ -168,7 +169,7 @@ public class DownloadHelper<T> {
             completionService.submit(getTask(i, threadSize, startFileIndex + i, endFileIndex, tableName, sortField, dao));
         }
         waitResult();
-        if (validateResult) {
+        if (validateResult != null && validateResult != NONE) {
             // 对结果进行校验，只校验一定的次数，防止因特殊原因，某些数据始终无法成功下载，陷入死循环
             int retry = 1;
             List<Integer> missingFiles;
@@ -177,7 +178,7 @@ public class DownloadHelper<T> {
                 logger.error("第 " + retry + " 次校验，下载遗失的文件：" + missingFiles);
                 List<DownloadHelper<T>> downloadHelpers = new ArrayList<>();
                 for (Integer missingFile : missingFiles) {
-                    downloadHelpers.add(new DownloadHelper<T>(1, missingFile, missingFile, tableName, sortField, dao, downloadService, false));
+                    downloadHelpers.add(new DownloadHelper<T>(1, missingFile, missingFile, tableName, sortField, dao, downloadService, NONE));
                 }
                 for (DownloadHelper downloadHelper : downloadHelpers) {
                     downloadHelper.getResult();
@@ -260,9 +261,21 @@ public class DownloadHelper<T> {
      * @return 未下载成功的文件
      */
     private List<Integer> validateResult() {
-        // 期待的文件数目
         int totalFiles = (int) Math.ceil(totalElements / (1.0 * SINGLE_FILE_MAX_SIZE));
-        final int expectedEndFileIndex = endFileIndex < totalFiles ? endFileIndex : totalFiles;
+        // 期待的起始、结束文件
+        final int expectStartFileIndex;
+        final int expectEndFileIndex;
+        if (validateResult == CURRENT) {
+            // 校验本次下载的文件
+            expectStartFileIndex = startFileIndex;
+            expectEndFileIndex = endFileIndex < totalFiles ? endFileIndex : totalFiles;
+        } else if (validateResult == ALL) {
+            // 校验所有文件
+            expectStartFileIndex = DEFAULT_START_FILE_INDEX;
+            expectEndFileIndex = totalFiles;
+        } else {
+            return null;
+        }
         List<Integer> missingFiles = new ArrayList<>();
 
         File[] files = new File(SearchUtils.getDataPath(tableName)).listFiles(new FileFilter() {
@@ -273,7 +286,7 @@ public class DownloadHelper<T> {
                     return false;
                 }
                 int id = parseName(pathname);
-                return id >= startFileIndex && id <= expectedEndFileIndex;
+                return id >= expectStartFileIndex && id <= expectEndFileIndex;
             }
         });
         if (ArrayUtils.isEmpty(files)) {
@@ -282,7 +295,7 @@ public class DownloadHelper<T> {
         // 将文件按名称排序
         sort(files);
         // 默认把所有文件视为未下载成功
-        for (int i = startFileIndex; i <= expectedEndFileIndex; i++) {
+        for (int i = expectStartFileIndex; i <= expectEndFileIndex; i++) {
             missingFiles.add(i);
         }
         result = 0L;
@@ -357,4 +370,20 @@ public class DownloadHelper<T> {
         files[j] = temp;
     }
 
+    public enum ValidateResult {
+        /**
+         * 不校验
+         */
+        NONE,
+        /**
+         * 校验所有文件
+         */
+        ALL,
+
+        /**
+         * 只校验本次下载的文件
+         */
+        CURRENT
+    }
+
 }

src/main/webapp/WEB-INF/views/console.html

@@ -86,8 +86,9 @@
 				<li>index/create?tableNames=product$kind,product$brand,product$component,trade$invoice_fmor,trade$order,trade$purchase,trade$invoice_fmpu,v$product$cmpgoods&componentFromFiles=false</li>
 				<li><a target="_blank">index/create</a></li>
 				<li><a target="_blank">index/create?tableNames=product$brand,trade$order</a></li>
-				<li><a target="_blank">index/multiDownloadComponent?threads=2&startFileIndex=0&endFileIndex=10000000</a></li>
-				<li><a target="_blank">index/multiDownloadGoods?threads=2&startFileIndex=0&endFileIndex=10000000</a></li>
+				<li><a target="_blank">index/multiDownloadComponent?threads=2&startFileIndex=0&endFileIndex=10000000&validateResult=current</a></li>
+				<li>index/multiDownloadGoods?threads=2&startFileIndex=0&endFileIndex=1&validateResult=all</li>
+				<li><a target="_blank">index/multiDownloadGoods?threads=2&startFileIndex=0&endFileIndex=1</a></li>
 				<li><a target="_blank">index/listen/start?interval=10</a></li>
 				<li><a target="_blank">index/listen/stop</a></li>
 				<li><a target="_blank">index/listen/restart</a></li>