Browse Source

fix duplicate data error in async download

sunyj 8 years ago
parent
commit
d47b7a2489
1 changed files with 11 additions and 9 deletions
  1. 11 9
      src/main/java/com/uas/search/support/DownloadService.java

+ 11 - 9
src/main/java/com/uas/search/support/DownloadService.java

@@ -92,24 +92,26 @@ public class DownloadService<T> {
                 size += content.size();
                 logger.info(name + " " + fileName + " - Downloaded..................." + size);
 
-                pageParams.setPage(pageParams.getPage() + 1);
-                pageInfo = new PageInfo(pageParams, sort);
-                pageResult = dao.findAll(pageInfo);
-                // 已经下载到最后一页
-                if (CollectionUtils.isEmpty(pageResult.getContent())) {
-                    break;
-                }
-
                 // 一个文件的函数达到指定值,写入新文件
                 if (count == SINGLE_FILE_MAX_SIZE) {
                     count = 0;
                     close(printWriter);
                     fileIndex += step;
+                    pageParams.setPage(pageParams.getPage() + step * SINGLE_FILE_MAX_SIZE / PAGE_SIZE);
+                    // 不创建多余文件
                     if (endFileIndex >= fileIndex) {
                         fileName = String.format("%010d", fileIndex) + ".txt";
                         file = new File(dataDir, fileName);
                         printWriter = new PrintWriter(file);
                     }
+                } else {
+                    pageParams.setPage(pageParams.getPage() + 1);
+                }
+                pageInfo = new PageInfo(pageParams, sort);
+                pageResult = dao.findAll(pageInfo);
+                // 已经下载到最后一页
+                if (CollectionUtils.isEmpty(pageResult.getContent())) {
+                    break;
                 }
             }
             logger.info(String.format("%s 下载完成,耗时%.2fs\n ", name, (new Date().getTime() - startTime) / 1000.0));
@@ -128,7 +130,7 @@ public class DownloadService<T> {
         return size;
     }
 
-    protected void println(PrintWriter printWriter, T element){
+    protected void println(PrintWriter printWriter, T element) {
         printWriter.println(JSONObject.toJSONString(element));
     }