增加器件转批次的接口

src/main/java/com/uas/search/controller/IndexController.java

@@ -60,6 +60,13 @@ public class IndexController {
         return "start download";
     }
 
+    @RequestMapping("/convertGoodsFromComponent")
+    @ResponseBody
+    public String convertGoodsFromComponent(HttpServletRequest request) {
+        return String.format("Downloaded in %.2fs",
+                indexService.convertGoodsFromComponent() / 1000.0);
+    }
+
     @RequestMapping("/multiDownloadGoods")
     @ResponseBody
     public String multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex, HttpServletRequest request) {

src/main/java/com/uas/search/service/IndexService.java

@@ -33,6 +33,13 @@ public interface IndexService {
      */
     public void multiDownloadComponent(Integer threads, Integer startFileIndex, Integer endFileIndex);
 
+    /**
+     * 根据本地器件的数据文件，转为批次数据
+     *
+     * @return 花费总时间 ms
+     */
+    public Long convertGoodsFromComponent();
+
     /**
      * 多线程下载批次的数据至本地文件中，以供建索引用
      *

src/main/java/com/uas/search/service/impl/IndexServiceImpl.java

@@ -1,5 +1,6 @@
 package com.uas.search.service.impl;
 
+import com.alibaba.fastjson.JSONException;
 import com.alibaba.fastjson.JSONObject;
 import com.uas.search.annotation.NotEmpty;
 import com.uas.search.constant.SearchConstants;
@@ -90,7 +91,7 @@ public class IndexServiceImpl implements IndexService {
 	/**
 	 * 单个文件存储的最大数据数目，需是PAGE_SIZE的整数倍
 	 */
-	public static final int SINGLE_FILE_MAX_SIZE = 100000;
+	public static final int SINGLE_FILE_MAX_SIZE = PAGE_SIZE;
 
 	private static Logger logger = LoggerFactory.getLogger(IndexServiceImpl.class);
 
@@ -114,39 +115,42 @@ public class IndexServiceImpl implements IndexService {
 			Long startTime = new Date().getTime();
 
 			for (String tableName : tableNames) {
-				Long start = new Date().getTime();
-				Long size = 0L;
-				deleteIndexes(tableName);
-				if (tableName.equals(SearchConstants.KIND_TABLE_NAME)) {
-					size = createKindIndexes();
-				} else if (tableName.equals(SearchConstants.BRAND_TABLE_NAME)) {
-					size = createBrandIndexes();
-				} else if (tableName.equals(SearchConstants.COMPONENT_TABLE_NAME)) {
-					// 只有明确指定不从本地文件读取数据时，才从数据库获取数据建立索引
-					if (componentFromFiles != null && !componentFromFiles) {
-						size = createComponentIndexes();
-					} else {
-						size = createComponentIndexesFromFiles();
-					}
-				} else if (tableName.equals(SearchConstants.GOODS_TABLE_NAME)) {
-					size = createGoodsIndexesFromFiles();
-				} else if (tableName.equals(SearchConstants.ORDER_TABLE_NAME)) {
-					size = createOrderIndexes();
-				} else if (tableName.equals(SearchConstants.ORDER_INVOICE_TABLE_NAME)) {
-					size = createOrderInvoiceIndexes();
-				} else if (tableName.equals(SearchConstants.PURCHASE_TABLE_NAME)) {
-					size = createPurchaseIndexes();
-				} else if (tableName.equals(SearchConstants.PURCHASE_INVOICE_TABLE_NAME)) {
-					size = createPurchaseInvoiceIndexes();
-				} else {
-					logger.error("不支持创建该表索引：" + tableName);
-					continue;
-				}
-				indexWriterManager.release(tableName);
-				indexSearcherManager.flushCache(tableName, indexWriter, null);
-				Long end = new Date().getTime();
-				logger.info(String.format("创建%s索引： %s条，耗时%.2fs\n ", tableName, size, (end - start) / 1000.0));
-			}
+                try {
+                    Long start = new Date().getTime();
+                    Long size = 0L;
+                    deleteIndexes(tableName);
+                    if (tableName.equals(SearchConstants.KIND_TABLE_NAME)) {
+                        size = createKindIndexes();
+                    } else if (tableName.equals(SearchConstants.BRAND_TABLE_NAME)) {
+                        size = createBrandIndexes();
+                    } else if (tableName.equals(SearchConstants.COMPONENT_TABLE_NAME)) {
+                        // 只有明确指定不从本地文件读取数据时，才从数据库获取数据建立索引
+                        if (componentFromFiles != null && !componentFromFiles) {
+                            size = createComponentIndexes();
+                        } else {
+                            size = createComponentIndexesFromFiles();
+                        }
+                    } else if (tableName.equals(SearchConstants.GOODS_TABLE_NAME)) {
+                        size = createGoodsIndexesFromFiles();
+                    } else if (tableName.equals(SearchConstants.ORDER_TABLE_NAME)) {
+                        size = createOrderIndexes();
+                    } else if (tableName.equals(SearchConstants.ORDER_INVOICE_TABLE_NAME)) {
+                        size = createOrderInvoiceIndexes();
+                    } else if (tableName.equals(SearchConstants.PURCHASE_TABLE_NAME)) {
+                        size = createPurchaseIndexes();
+                    } else if (tableName.equals(SearchConstants.PURCHASE_INVOICE_TABLE_NAME)) {
+                        size = createPurchaseInvoiceIndexes();
+                    } else {
+                        logger.error("不支持创建该表索引：" + tableName);
+                        continue;
+                    }
+                    indexSearcherManager.flushCache(tableName, indexWriter, null);
+                    Long end = new Date().getTime();
+                    logger.info(String.format("创建%s索引： %s条，耗时%.2fs\n ", tableName, size, (end - start) / 1000.0));
+                }finally {
+                    indexWriterManager.release(tableName);
+                }
+            }
 
 			Long endTime = new Date().getTime();
 			logger.info(String.format("索引创建成功， 共用时间%.2fs\n", (endTime - startTime) / 1000.0));
@@ -200,47 +204,51 @@ public class IndexServiceImpl implements IndexService {
 		return createIndexesWithObjects(brands.toArray());
 	}
 
-	private Long createComponentIndexesFromFiles() {
-		logger.info("正在创建器件索引...");
-		Long size = 0L;
-		try {
-			// 从本地路径读取器件数据
-			String componentDataPath = SearchUtils.getDataPath(SearchConstants.COMPONENT_TABLE_NAME);
-			File[] files = new File(componentDataPath).listFiles();
-			if (files == null || files.length == 0) {
-				logger.info("创建器件索引失败，原因：器件数据文件不存在！");
-				return 0L;
-			}
-			// 将要创建的索引总数目约为：文件数目*单个文件的行数
-			long totalSize = files.length * SINGLE_FILE_MAX_SIZE;
-			for (File file : files) {
-				logger.info("读取文件: " + file.getName());
-				BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
-				String line = null;
-				while (!StringUtils.isEmpty(line = bufferedReader.readLine())) {
-					Component component = JSONObject.parseObject(line, Component.class);
-					Document document = ObjectToDocumentUtils.toDocument(component);
-					if (document != null) {
-						size++;
-						// 每创建10000条，打印一次进度
-						if (size % 10000 == 0) {
-							logger.info(String.format("Component indexed...................%.2f%%",
-									size * 100.0 / totalSize));
-						}
-						indexWriter.addDocument(document);
-					}
-				}
-				indexWriter.commit();
-				bufferedReader.close();
-			}
-		} catch (FileNotFoundException e) {
-			logger.error("创建器件索引失败，原因：器件数据文件不存在！");
-			return 0L;
-		} catch (IOException e) {
-			logger.error("", e);
-		}
-		return size;
-	}
+    private Long createComponentIndexesFromFiles() throws IOException {
+        logger.info("正在创建器件索引...");
+        Long size = 0L;
+        BufferedReader bufferedReader = null;
+        try {
+            // 从本地路径读取器件数据
+            String componentDataPath = SearchUtils.getDataPath(SearchConstants.COMPONENT_TABLE_NAME);
+            File[] files = new File(componentDataPath).listFiles();
+            if (files == null || files.length == 0) {
+                logger.info("创建器件索引失败，原因：器件数据文件不存在！");
+                return 0L;
+            }
+            for (File file : files) {
+                logger.info("读取文件: " + file.getName());
+                bufferedReader = new BufferedReader(new FileReader(file));
+                String line;
+                while (!StringUtils.isEmpty(line = bufferedReader.readLine())) {
+                    Component component;
+                    try {
+                        component = JSONObject.parseObject(line, Component.class);
+                    } catch (JSONException e) {
+                        throw new IllegalArgumentException(line, e);
+                    }
+                    Document document = ObjectToDocumentUtils.toDocument(component);
+                    if (document != null) {
+                        size++;
+                        // 每创建10000条，打印一次进度
+                        if (size % 10000 == 0) {
+                            logger.info("Component indexed..................." + size);
+                        }
+                        indexWriter.addDocument(document);
+                    }
+                }
+                indexWriter.commit();
+            }
+        } catch (FileNotFoundException e) {
+            logger.error("创建器件索引失败，原因：器件数据文件不存在！");
+            return 0L;
+        }  finally {
+            if (bufferedReader != null) {
+                bufferedReader.close();
+            }
+        }
+        return size;
+    }
 
 	/**
 	 * 创建器件索引，从数据库取数据
@@ -290,9 +298,10 @@ public class IndexServiceImpl implements IndexService {
 		return size;
 	}
 
-	private Long createGoodsIndexesFromFiles() {
+	private Long createGoodsIndexesFromFiles() throws IOException {
 		logger.info("正在创建批次索引...");
 		Long size = 0L;
+        BufferedReader bufferedReader=null;
 		try {
 			// 从本地路径读取批次数据
 			File[] files = new File(SearchUtils.getDataPath(SearchConstants.GOODS_TABLE_NAME)).listFiles();
@@ -300,34 +309,39 @@ public class IndexServiceImpl implements IndexService {
 				logger.info("创建批次索引失败，原因：批次数据文件不存在！");
 				return 0L;
 			}
-			// 将要创建的索引总数目约为：文件数目*单个文件的行数
-			long totalSize = files.length * SINGLE_FILE_MAX_SIZE;
 			for (File file : files) {
 				logger.info("读取文件: " + file.getName());
-				BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
-				String line = null;
+				bufferedReader = new BufferedReader(new FileReader(file));
+				String line;
 				while (!StringUtils.isEmpty(line = bufferedReader.readLine())) {
-					Goods goods = JSONObject.parseObject(line, Goods.class);
-					Document document = ObjectToDocumentUtils.toDocument(goods);
+                    Goods goods;
+                    try {
+                        goods = JSONObject.parseObject(line, Goods.class);
+                    } catch (JSONException e) {
+                        throw new IllegalArgumentException(line, e);
+                    }
+                    Document document = ObjectToDocumentUtils.toDocument(goods);
 					if (document != null) {
 						size++;
 						// 每创建10000条，打印一次进度
 						if (size % 10000 == 0) {
-							logger.info(
-									String.format("Goods indexed...................%.2f%%", size * 100.0 / totalSize));
+                            logger.info("Goods indexed..................." + size);
 						}
 						indexWriter.addDocument(document);
 					}
 				}
 				indexWriter.commit();
-				bufferedReader.close();
+                bufferedReader.close();
 			}
 		} catch (FileNotFoundException e) {
 			logger.error("创建批次索引失败，原因：批次数据文件不存在！");
 			return 0L;
-		} catch (IOException e) {
-			logger.error("", e);
-		}
+		} finally {
+            indexWriter.commit();
+            if (bufferedReader != null) {
+                bufferedReader.close();
+            }
+        }
 		return size;
 	}
 
@@ -398,6 +412,71 @@ public class IndexServiceImpl implements IndexService {
         }
     }
 
+    @Override
+    public Long convertGoodsFromComponent() {
+        Long startTime = new Date().getTime();
+        Long size = 0L;
+        logger.info("转换批次... ");
+        BufferedReader bufferedReader = null;
+        try {
+            // 从本地路径读取器件数据
+            String componentDataPath = SearchUtils.getDataPath(SearchConstants.COMPONENT_TABLE_NAME);
+            String goodsDataPath = SearchUtils.getDataPath(SearchConstants.GOODS_TABLE_NAME);
+            File[] files = new File(componentDataPath).listFiles();
+            if (files == null || files.length == 0) {
+                logger.info("转换批次失败，原因：器件数据文件不存在！");
+                return 0L;
+            }
+            int fileIndex = 1;
+            for (File file : files) {
+                logger.info("读取器件文件: " + file.getName());
+                bufferedReader = new BufferedReader(new FileReader(file));
+                String goodsFileName = String.format("%05d", fileIndex) + ".txt";
+                PrintWriter printWriter = new PrintWriter(goodsDataPath + "/" + goodsFileName);
+                String line;
+                while (!StringUtils.isEmpty(line = bufferedReader.readLine())) {
+                    Component component;
+                    try {
+                        component = JSONObject.parseObject(line, Component.class);
+                    } catch (JSONException e) {
+                        throw new IllegalArgumentException(line, e);
+                    }
+                    // 器件作为主体，得到批次
+                    printWriter.println(JSONObject.toJSONString(new Goods(null, null, component)));
+                    size++;
+                }
+                logger.info(goodsFileName + " - Converted..................." + size);
+                printWriter.flush();
+                printWriter.close();
+                fileIndex++;
+                bufferedReader.close();
+            }
+            long endStartTime = new Date().getTime();
+            logger.info(String.format("转换完成，耗时%.2fs\n ", (endStartTime - startTime) / 1000.0));
+            return endStartTime - startTime;
+        } catch (Throwable e) {
+            throw new IllegalStateException("批次转换失败", e);
+        }finally{
+            if(bufferedReader!=null){
+                try {
+                    bufferedReader.close();
+                } catch (IOException e) {
+                    throw new IllegalStateException("批次转换失败", e);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex) {
+        threads = threads == null || threads < 1 ? 1 : threads;
+        startFileIndex = startFileIndex == null || startFileIndex < 1 ? 1 : startFileIndex;
+        endFileIndex = endFileIndex == null || endFileIndex < 1 ? 1024 * 1024 * 1024 : endFileIndex;
+        for (int i = 1; i <= threads; i++) {
+            new Thread(new DownloadGoodsTread("Thread-" + i, threads, startFileIndex + i - 1, endFileIndex)).start();
+        }
+    }
+
     /**
      * 下载器件的线程
      */
@@ -461,13 +540,14 @@ public class IndexServiceImpl implements IndexService {
                     file.mkdirs();
                 }
                 while (totalPages >= startFileIndex && endFileIndex >= startFileIndex) {
-                    PrintWriter printWriter = new PrintWriter(goodsDataPath + "/" + String.format("%05d", startFileIndex) + ".txt");
+                    String componentFileName = String.format("%05d", startFileIndex) + ".txt";
+                    PrintWriter printWriter = new PrintWriter(goodsDataPath + "/" + componentFileName);
                     List<Component> content = pageResult.getContent();
                     for (Component element : content) {
                         printWriter.println(JSONObject.toJSONString(element));
                     }
                     size += content.size();
-                    logger.info(name + " " + startFileIndex + ".txt" + " - Downloaded..................." + size);
+                    logger.info(name + " " + componentFileName + " - Downloaded..................." + size);
 
                     printWriter.flush();
                     printWriter.close();
@@ -477,24 +557,13 @@ public class IndexServiceImpl implements IndexService {
                     pageResult = componentDao.findAll(pageInfo);
                 }
 
-                Long endTime = new Date().getTime();
-                logger.info(String.format("%s 下载完成，耗时%.2fs\n ", name, (endTime - startTime) / 1000.0));
+                logger.info(String.format("%s 下载完成，耗时%.2fs\n ", name, (new Date().getTime() - startTime) / 1000.0));
             } catch (Throwable e) {
-                logger.error(name, e);
+                logger.error(name + " 器件下载失败", e);
             }
         }
     }
 
-    @Override
-    public void multiDownloadGoods(Integer threads, Integer startFileIndex, Integer endFileIndex) {
-        threads = threads == null || threads < 1 ? 1 : threads;
-        startFileIndex = startFileIndex == null || startFileIndex < 1 ? 1 : startFileIndex;
-        endFileIndex = endFileIndex == null || endFileIndex < 1 ? 1024 * 1024 * 1024 : endFileIndex;
-        for (int i = 1; i <= threads; i++) {
-            new Thread(new DownloadGoodsTread("Thread-" + i, threads, startFileIndex + i - 1, endFileIndex)).start();
-        }
-    }
-
     /**
      * 下载批次的线程
      */
@@ -558,7 +627,8 @@ public class IndexServiceImpl implements IndexService {
                     file.mkdirs();
                 }
                 while (totalPages >= startFileIndex && endFileIndex >= startFileIndex) {
-                    PrintWriter printWriter = new PrintWriter(goodsDataPath + "/" + String.format("%05d", startFileIndex) + ".txt");
+                    String goodsFileName = String.format("%05d", startFileIndex) + ".txt";
+                    PrintWriter printWriter = new PrintWriter(goodsDataPath + "/" + goodsFileName);
                     List<Component> content = pageResult.getContent();
                     for (Component element : content) {
                         // 器件作为主体，再下载批次
@@ -568,7 +638,7 @@ public class IndexServiceImpl implements IndexService {
                         }
                     }
                     size += content.size();
-                    logger.info(name + " " + startFileIndex + ".txt" + " - Downloaded..................." + size);
+                    logger.info(name + " " + goodsFileName + " - Downloaded..................." + size);
 
                     printWriter.flush();
                     printWriter.close();
@@ -578,10 +648,9 @@ public class IndexServiceImpl implements IndexService {
                     pageResult = componentDao.findAll(pageInfo);
                 }
 
-                Long endTime = new Date().getTime();
-                logger.info(String.format("%s 下载完成，耗时%.2fs\n ", name, (endTime - startTime) / 1000.0));
+                logger.info(String.format("%s 下载完成，耗时%.2fs\n ", name, (new Date().getTime() - startTime) / 1000.0));
             } catch (Throwable e) {
-                logger.error(name, e);
+                logger.error(name + " 批次下载失败", e);
             }
         }
     }

src/main/webapp/WEB-INF/views/console.html

@@ -87,6 +87,7 @@
 				<li><a target="_blank">index/create</a></li>
 				<li><a target="_blank">index/create?tableNames=product$brand,trade$order</a></li>
 				<li><a target="_blank">index/multiDownloadComponent?threads=2&startFileIndex=0&endFileIndex=10000000</a></li>
+                <li><a target="_blank">index/convertGoodsFromComponent</a></li>
 				<li><a target="_blank">index/multiDownloadGoods?threads=2&startFileIndex=0&endFileIndex=10000000</a></li>
 				<li><a target="_blank">index/listen/start?interval=10</a></li>
 				<li><a target="_blank">index/listen/stop</a></li>