下载优化调整

mall-search/src/main/java/com/uas/search/service/JdbcService.java

@@ -1,6 +1,9 @@
 package com.uas.search.service;
 
 import java.util.List;
+import java.util.Map;
+
+import com.alibaba.fastjson.JSONObject;
 import org.springframework.data.domain.Sort;
 
 /**
@@ -28,4 +31,8 @@ public interface JdbcService {
      * @return
      */
     List<Long> getLimitId(String tablename, String direction, String sortField, Long start, Integer size);
+
+    String abstractTransform(Class className, String property) throws NoSuchFieldException;
+
+    List<Object> getData(String tableName, Long startId, Long endId, String sortField, String direct);
 }

mall-search/src/main/java/com/uas/search/service/impl/IndexServiceImpl.java

@@ -43,6 +43,7 @@ import com.uas.search.model.PurchaseInvoice;
 import com.uas.search.model.TradeGoods;
 import com.uas.search.model.V_Products;
 import com.uas.search.service.IndexService;
+import com.uas.search.service.JdbcService;
 import com.uas.search.service.SearchService;
 import com.uas.search.support.DownloadHelper;
 import com.uas.search.support.DownloadService;
@@ -136,6 +137,8 @@ public class IndexServiceImpl implements IndexService {
     private SearchService searchService;
     @Autowired
     private V_ProductsDao v_productsDao;
+    @Autowired
+    private JdbcService jdbcService;
     /**
      * 是否正在创建索引
      */
@@ -792,19 +795,23 @@ public class IndexServiceImpl implements IndexService {
             throw new IllegalArgumentException("线程数量不可超过 " + druidDBConfiguration.getMaxActive());
         }
 
+        String sortField = null;
         if (tableName.equals(COMPONENT_TABLE_NAME)) {
             try {
-                Field id = Component.class.getDeclaredField("id");
-                Annotation annotation = id.getAnnotation(Column.class);
-                String columnName = ((Column) annotation).name();
+                sortField = jdbcService.abstractTransform(Component.class, "id");
             } catch (NoSuchFieldException e) {
 
             }
-            DownloadHelper<Component> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id", componentDao, new DownloadService<Component>(), validateResult);
+            DownloadHelper<Component> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, sortField, Component.class, new DownloadService<Component>(), validateResult);
             return downloadHelper.getResult();
 
         } else if (tableName.equals(GOODS_TABLE_NAME)) {
-            DownloadHelper<TradeGoods> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id", tradeGoodsDao, new DownloadService<TradeGoods>() {
+            try {
+                sortField = jdbcService.abstractTransform(TradeGoods.class, "id");
+            } catch (NoSuchFieldException e) {
+
+            }
+            DownloadHelper<TradeGoods> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, sortField, TradeGoods.class, new DownloadService<TradeGoods>() {
                 @Override
                 protected void println(PrintWriter printWriter, TradeGoods element) {
                     Goods goods = goodsDao.findByTradeGoods(element);
@@ -813,11 +820,21 @@ public class IndexServiceImpl implements IndexService {
             }, validateResult);
             return downloadHelper.getResult();
         } else if (tableName.equals(PCB_TABLE_NAME)) {
-            DownloadHelper<PCB> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id", pcbDao, new DownloadService<PCB>(), validateResult);
+            try {
+                sortField = jdbcService.abstractTransform(PCB.class, "id");
+            } catch (NoSuchFieldException e) {
+
+            }
+            DownloadHelper<PCB> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, sortField, PCB.class, new DownloadService<PCB>(), validateResult);
             return downloadHelper.getResult();
 
         } else if (tableName.equals(PCB_GOODS_TABLE_NAME)) {
-            DownloadHelper<TradeGoods> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id", tradeGoodsDao, new DownloadService<TradeGoods>() {
+            try {
+                sortField = jdbcService.abstractTransform(TradeGoods.class, "id");
+            } catch (NoSuchFieldException e) {
+
+            }
+            DownloadHelper<TradeGoods> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, sortField, TradeGoods.class, new DownloadService<TradeGoods>() {
                 @Override
                 protected void println(PrintWriter printWriter, TradeGoods element) {
                     PCBGoods pcbGoods = pcbGoodsDao.findByTradeGoods(element);
@@ -826,7 +843,12 @@ public class IndexServiceImpl implements IndexService {
             }, validateResult);
             return downloadHelper.getResult();
         } else if (tableName.equals(PRODUCTS_PRIVATE_TABLE_NAME)) {
-            DownloadHelper<V_Products> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, "id", v_productsDao, new DownloadService<V_Products>() {}, validateResult);
+            try {
+                sortField = jdbcService.abstractTransform(V_Products.class, "id");
+            } catch (NoSuchFieldException e) {
+
+            }
+            DownloadHelper<V_Products> downloadHelper = new DownloadHelper<>(threads, startFileIndex, endFileIndex, tableName, sortField, V_Products.class, new DownloadService<V_Products>() {}, validateResult);
             return downloadHelper.getResult();
         } else {
             throw new IllegalArgumentException("多线程下载不支持该表：" + tableName);

mall-search/src/main/java/com/uas/search/service/impl/JdbcServiceImpl.java

@@ -1,11 +1,21 @@
 package com.uas.search.service.impl;
 
+import com.alibaba.fastjson.JSONObject;
+import com.uas.search.model.Component;
 import com.uas.search.service.JdbcService;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.jdbc.core.JdbcTemplate;
 
+import javax.persistence.Column;
+import javax.persistence.Table;
+
 /**
  * Created by wangyc on 2018/6/11.
  *
@@ -30,4 +40,18 @@ public class JdbcServiceImpl implements JdbcService {
             tablename, sortField, start + size - 1), Long.class));
         return ids;
     }
+
+    @Override
+    public String abstractTransform(Class className, String property) throws NoSuchFieldException {
+        Field id = className.getDeclaredField("id");
+        Annotation annotation = id.getAnnotation(Column.class);
+        return ((Column) annotation).name();
+    }
+
+    @Override
+    public List<Object> getData(String tableName, Long startId, Long endId, String sortField,
+        String direct) {
+        String sql = String.format("select * from %s order by %s %s limit %s between %l and %l", tableName, sortField, direct, sortField, startId, endId);
+        return jdbcTemplate.queryForList(sql, Object.class);
+    }
 }

mall-search/src/main/java/com/uas/search/support/DownloadHelper.java

@@ -76,9 +76,9 @@ public class DownloadHelper<T> {
     private String sortField;
 
     /**
-     * dao
+     * class
      */
-    private JpaRepository<T, Long> dao;
+    private Class className;
 
     /**
      * 下载的实现
@@ -122,11 +122,11 @@ public class DownloadHelper<T> {
      * @param endFileIndex    结束的文件
      * @param tableName       要下载的表
      * @param sortField       排序字段
-     * @param dao             dao
+     * @param className       转换class
      * @param downloadService 下载的实现
      */
-    public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, JpaRepository<T, Long> dao, DownloadService<T> downloadService) {
-        this(threadSize, startFileIndex, endFileIndex, tableName, sortField, dao, downloadService, CURRENT);
+    public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, Class className, DownloadService<T> downloadService) {
+        this(threadSize, startFileIndex, endFileIndex, tableName, sortField, className, downloadService, CURRENT);
     }
 
     /**
@@ -135,11 +135,11 @@ public class DownloadHelper<T> {
      * @param endFileIndex    结束的文件
      * @param tableName       要下载的表
      * @param sortField       排序字段
-     * @param dao             dao
+     * @param className       转换class
      * @param downloadService 下载的实现
      * @param validateResult  下载完成后，是否对结果进行校验
      */
-    public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, JpaRepository<T, Long> dao, DownloadService<T> downloadService, ValidateResult validateResult) {
+    public DownloadHelper(Integer threadSize, Integer startFileIndex, Integer endFileIndex, String tableName, String sortField, Class className, DownloadService<T> downloadService, ValidateResult validateResult) {
         if (threadSize == null || threadSize < MIN_THREAD_SIZE || threadSize > MAX_THREAD_SIZE) {
             throw new IllegalArgumentException("threadSize is between " + MIN_THREAD_SIZE + " and " + MAX_THREAD_SIZE);
         }
@@ -152,7 +152,7 @@ public class DownloadHelper<T> {
         this.endFileIndex = endFileIndex == null || endFileIndex < DEFAULT_START_FILE_INDEX ? DEFAULT_END_FILE_INDEX : endFileIndex;
         this.tableName = tableName;
         this.sortField = sortField;
-        this.dao = dao;
+        this.className = className;
         this.validateResult = validateResult;
         start();
     }
@@ -167,12 +167,13 @@ public class DownloadHelper<T> {
             // 删除旧的文件
             FileUtils.deleteSubFiles(new File(SearchUtils.getDataPath(tableName)));
         }
+        // 获取下载总条数
         getTotalElements(tableName);
         logger.info(tableName + " 发现数据 " + totalElements + " 条");
         // 线程数量不可高于下载的文件数量
         threadSize = threadSize <= endFileIndex - startFileIndex + 1 ? threadSize : endFileIndex - startFileIndex + 1;
         for (int i = 0; i < threadSize; i++) {
-            completionService.submit(getTask(i, threadSize, startFileIndex + i, endFileIndex, tableName, sortField, dao));
+            completionService.submit(getTask(i, threadSize, startFileIndex + i, endFileIndex, tableName, sortField, className));
         }
         waitResult();
         if (validateResult != null && validateResult != NONE) {
@@ -184,7 +185,7 @@ public class DownloadHelper<T> {
                 logger.error("第 " + retry + " 次校验，下载遗失的文件：" + missingFiles);
                 List<DownloadHelper<T>> downloadHelpers = new ArrayList<>();
                 for (Integer missingFile : missingFiles) {
-                    downloadHelpers.add(new DownloadHelper<>(1, missingFile, missingFile, tableName, sortField, dao, downloadService, NONE));
+                    downloadHelpers.add(new DownloadHelper<>(1, missingFile, missingFile, tableName, sortField, className, downloadService, NONE));
                 }
                 for (DownloadHelper downloadHelper : downloadHelpers) {
                     downloadHelper.getResult();
@@ -204,7 +205,7 @@ public class DownloadHelper<T> {
      * @param tableName 表名
      */
     private void getTotalElements(String tableName) {
-        totalElements = jdbcTemplate.queryForObject("select count(1) from " + tableName, Long.class);
+        totalElements = jdbcTemplate.queryForObject(String.format("select count(1) from %s", tableName), Long.class);
 //        PageParams pageParams = new PageParams();
 //        pageParams.setPage(startFileIndex);
 //        pageParams.setSize(PAGE_SIZE);
@@ -221,14 +222,14 @@ public class DownloadHelper<T> {
      * @param endFileIndex   结束的文件
      * @param tableName      要下载的表
      * @param sortField      排序字段
-     * @param dao            dao
+     * @param className       转换class
      * @return 任务
      */
-    private Callable<Long> getTask(final int id, final int step, final int startFileIndex, final int endFileIndex, final String tableName, final String sortField, final JpaRepository<T, Long> dao) {
+    private Callable<Long> getTask(final int id, final int step, final int startFileIndex, final int endFileIndex, final String tableName, final String sortField, final Class className) {
         return new Callable<Long>() {
             @Override
             public Long call() throws Exception {
-                return downloadService.download(id, step, startFileIndex, endFileIndex, tableName, sortField, dao);
+                return downloadService.download(id, step, startFileIndex, endFileIndex, tableName, sortField, className);
             }
         };
     }

mall-search/src/main/java/com/uas/search/support/DownloadService.java

@@ -3,6 +3,7 @@ package com.uas.search.support;
 import static com.uas.search.service.impl.IndexServiceImpl.PAGE_SIZE;
 import static com.uas.search.service.impl.IndexServiceImpl.SINGLE_FILE_MAX_SIZE;
 
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.uas.search.constant.model.PageInfo;
 import com.uas.search.constant.model.PageParams;
@@ -16,9 +17,7 @@ import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Sort;
-import org.springframework.data.jpa.repository.JpaRepository;
 
 /**
  * 下载线程
@@ -38,10 +37,10 @@ public class DownloadService<T> {
      * @param endFileIndex   结束的文件
      * @param tableName      要下载的表
      * @param sortField      排序字段
-     * @param dao            dao
+     * @param className      转换class
      * @return 下载的数量
      */
-    public long download(int id, int step, int startFileIndex, int endFileIndex, String tableName, String sortField, JpaRepository<T, Long> dao) {
+    public long download(int id, int step, int startFileIndex, int endFileIndex, String tableName, String sortField, Class className) {
         long size = 0L;
         Logger logger = LoggerFactory.getLogger(getClass());
         String name = "Thread_" + id + "_" + startFileIndex + "-" + endFileIndex;
@@ -58,16 +57,17 @@ public class DownloadService<T> {
             Long startTime = System.currentTimeMillis();
             logger.info(name + " 下载 " + tableName + "...");
 
-            Sort sort = new Sort(Sort.Direction.ASC, sortField);
+//            Sort sort = new Sort(Sort.Direction.ASC, sortField);
             // 分页获取数据
             PageParams pageParams = new PageParams();
             pageParams.setPage((startFileIndex - 1) * SINGLE_FILE_MAX_SIZE / PAGE_SIZE + 1);
             pageParams.setSize(PAGE_SIZE);
-            PageInfo pageInfo = new PageInfo(pageParams, sort);
-            Page<T> pageResult = dao.findAll(pageInfo);
-
-            jdbcService.getLimitId(tableName, Sort.Direction.ASC.name(), sortField, (long) (startFileIndex - 1) * SINGLE_FILE_MAX_SIZE, PAGE_SIZE);
+            PageInfo pageInfo = new PageInfo(pageParams);
+//            Page<T> pageResult = dao.findAll(pageInfo);
 
+            List<Long> ids = jdbcService.getLimitId(tableName, Sort.Direction.ASC.name(), sortField, (long) (startFileIndex - 1) * SINGLE_FILE_MAX_SIZE, PAGE_SIZE);
+            // TODO 下载
+            List<T> data = JSONArray.parseArray(jdbcService.getData(tableName, ids.get(0), ids.get(1), sortField, Sort.Direction.ASC.name()).toString(), className);
             int totalFiles = (int) Math.ceil(jdbcService.getTotalElements(tableName) / (1.0 * SINGLE_FILE_MAX_SIZE));
             if (startFileIndex > totalFiles) {
                 logger.info(name + " startFileIndex 超过 " + totalFiles);
@@ -89,7 +89,7 @@ public class DownloadService<T> {
             int count = 0;
             while (endFileIndex >= fileIndex) {
                 // 数据以 JSON 格式写入文件
-                List<T> content = pageResult.getContent();
+                List<T> content = data;
                 for (T element : content) {
                     println(printWriter, element);
                 }
@@ -112,10 +112,10 @@ public class DownloadService<T> {
                 } else {
                     pageParams.setPage(pageParams.getPage() + 1);
                 }
-                pageInfo = new PageInfo(pageParams, sort);
-                pageResult = dao.findAll(pageInfo);
+//                pageInfo = new PageInfo(pageParams, sort);
+//                pageResult = dao.findAll(pageInfo);
                 // 已经下载到最后一页
-                if (CollectionUtils.isEmpty(pageResult.getContent())) {
+                if (CollectionUtils.isEmpty(data)) {
                     break;
                 }
             }
@@ -146,4 +146,4 @@ public class DownloadService<T> {
         }
     }
 
-}
+}