增加两个接口：获取索引中的所有数据；获取索引中的数据写入json文件

search-console/src/main/java/com/uas/search/console/controller/SearchController.java

@@ -1,17 +1,24 @@
 package com.uas.search.console.controller;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.util.CollectionUtils;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 
 import com.alibaba.druid.util.StringUtils;
@@ -24,9 +31,14 @@ import com.uas.search.console.model.ComponentSimpleInfo;
 import com.uas.search.console.model.GoodsSimpleInfo;
 import com.uas.search.console.model.KindSimpleInfo;
 import com.uas.search.console.service.InnerSearchService;
+import com.uas.search.console.service.impl.IndexServiceImpl;
+import com.uas.search.console.util.FileUtils;
+import com.uas.search.console.util.SearchUtils;
+import com.uas.search.exception.SearchException;
 import com.uas.search.model.CollectField;
 import com.uas.search.model.PageParams;
 import com.uas.search.model.PageParams.FilterField;
+import com.uas.search.model.SPage;
 import com.uas.search.model.Sort;
 import com.uas.search.model.Sort.Field;
 import com.uas.search.service.SearchService;
@@ -47,6 +59,8 @@ public class SearchController {
 	private InnerSearchService innerSearchService = ContextUtils.getApplicationContext().getBean("searchServiceImpl",
 			InnerSearchService.class);
 
+	private Logger logger = LoggerFactory.getLogger(getClass());
+
 	@RequestMapping("/kindIds/{keyword}")
 	@ResponseBody
 	public List<Long> seachKindIds(@PathVariable String keyword) {
@@ -288,4 +302,67 @@ public class SearchController {
 	public GoodsSimpleInfo getGoods(@PathVariable String id) {
 		return innerSearchService.getGoods(id);
 	}
+
+	@RequestMapping("/allObjects")
+	@ResponseBody
+	public SPage<Object> getAllObjects(@RequestParam(required = true) String tableName, int page, int size) {
+		return innerSearchService.getAllObjects(tableName.toLowerCase(), page, size);
+	}
+
+	@RequestMapping("/allObjectsToFiles")
+	@ResponseBody
+	public String writeAllObjectsToFiles(@RequestParam(required = true) String tableName) {
+		int page = 1;
+		int size = 1000;
+		// 不能边更新索引边分页获取索引中的数据，因为索引更新后，分页顺序可能也会变化，
+		// 所以要先把数据保存到本地，等待全部获取之后重建索引
+		Long startTime = new Date().getTime();
+		// 先删除旧的文件
+		FileUtils.deleteSubFiles(new File(SearchUtils.getDataPath(tableName)));
+
+		SPage<Object> sPage = innerSearchService.getAllObjects(tableName.toLowerCase(), page, size);
+		// 索引中数据的总数目
+		long totalElements = sPage.getTotalElement();
+		logger.info("发现数据：" + totalElements + "条");
+		int fileIndex = 1;
+		PrintWriter printWriter = null;
+		int count = 0;
+		try {
+			File file = new File(SearchUtils.getDataPath(tableName));
+			if (!file.exists()) {
+				file.mkdirs();
+			}
+			printWriter = new PrintWriter(SearchUtils.getDataPath(tableName) + "/" + fileIndex + ".txt");
+			while (true) {
+				// 一个文件存放100000条数据，一旦超过，写入新的文件
+				if (count > IndexServiceImpl.SINGLE_FILE_MAX_SIZE) {
+					count = 1;
+					printWriter.flush();
+					printWriter.close();
+					fileIndex++;
+					printWriter = new PrintWriter(SearchUtils.getDataPath(tableName) + "/" + fileIndex + ".txt");
+				}
+				List<Object> content = sPage.getContent();
+				for (Object element : content) {
+					printWriter.println(JSONObject.toJSONString(element));
+					count++;
+				}
+				logger.info(String.format(tableName + "...................%.2f%%",
+						(page - 1) * size * 100.0 / totalElements));
+
+				if (++page > sPage.getTotalPage()) {
+					break;
+				}
+				sPage = innerSearchService.getAllObjects(tableName.toLowerCase(), page, size);
+			}
+			printWriter.flush();
+			printWriter.close();
+		} catch (FileNotFoundException | SecurityException | IllegalArgumentException e) {
+			throw new SearchException(e).setDetailedMessage(e);
+		}
+		Long endTime = new Date().getTime();
+		String message = String.format("写入数据%s条，耗时%.2fs\n ", totalElements, (endTime - startTime) / 1000.0);
+		logger.info(message);
+		return message;
+	}
 }

search-console/src/main/java/com/uas/search/console/service/InnerSearchService.java

@@ -4,6 +4,7 @@ import com.uas.search.console.model.BrandSimpleInfo;
 import com.uas.search.console.model.ComponentSimpleInfo;
 import com.uas.search.console.model.GoodsSimpleInfo;
 import com.uas.search.console.model.KindSimpleInfo;
+import com.uas.search.model.SPage;
 
 /**
  * search-console下自用搜索接口
@@ -43,4 +44,14 @@ public interface InnerSearchService {
 	 * @return
 	 */
 	public GoodsSimpleInfo getGoods(String id);
+
+	/**
+	 * 分页获取本地指定表的索引中的所有数据
+	 * 
+	 * @param tableName
+	 * @param page
+	 * @param size
+	 * @return
+	 */
+	public SPage<Object> getAllObjects(String tableName, int page, int size);
 }

search-console/src/main/java/com/uas/search/console/service/impl/SearchServiceImpl.java

@@ -1197,4 +1197,16 @@ public class SearchServiceImpl implements SearchService, InnerSearchService {
 				SearchUtils.getDocumentById(SearchConstants.GOODS_TABLE_NAME, SearchConstants.GOODS_ID_FIELD, id));
 	}
 
+	@Override
+	public SPage<Object> getAllObjects(String tableName, int page, int size) {
+		SPage<Document> documents = SearchUtils.getAllDocuments(tableName, page, size);
+		SPage<Object> sPage = new SPage<Object>(documents.getTotalPage(), documents.getTotalElement(),
+				documents.getPage(), documents.getSize(), documents.isFirst(), documents.isLast());
+		List<Object> content = new ArrayList<>();
+		for (Document document : documents.getContent()) {
+			content.add(DocumentToObjectUtils.toObject(document, tableName));
+		}
+		return sPage;
+	}
+
 }

search-console/src/main/java/com/uas/search/console/util/DocumentToObjectUtils.java

@@ -25,6 +25,7 @@ import com.uas.search.console.model.PurchaseInvoiceDetailSimpleInfo;
 import com.uas.search.console.model.PurchaseInvoiceSimpleInfo;
 import com.uas.search.console.model.PurchaseSimpleInfo;
 import com.uas.search.console.model.StoreSimpleInfo;
+import com.uas.search.exception.SearchException;
 import com.uas.search.model.BaseOrderDetail;
 
 /**
@@ -35,6 +36,33 @@ import com.uas.search.model.BaseOrderDetail;
  */
 public class DocumentToObjectUtils {
 
+	/**
+	 * 将Document转换为对象
+	 * 
+	 * @param document
+	 * @return
+	 */
+	public static Object toObject(Document document, String tableName) {
+		if (tableName.equals(SearchConstants.KIND_TABLE_NAME)) {
+			return toKind(document);
+		} else if (tableName.equals(SearchConstants.BRAND_TABLE_NAME)) {
+			return toBrand(document);
+		} else if (tableName.equals(SearchConstants.COMPONENT_TABLE_NAME)) {
+			return toComponent(document);
+		} else if (tableName.equals(SearchConstants.GOODS_TABLE_NAME)) {
+			return toGoods(document);
+		} else if (tableName.equals(SearchConstants.ORDER_TABLE_NAME)) {
+			return toOrder(document);
+		} else if (tableName.equals(SearchConstants.ORDER_INVOICE_TABLE_NAME)) {
+			return toOrderInvoice(document);
+		} else if (tableName.equals(SearchConstants.PURCHASE_TABLE_NAME)) {
+			return toPurchase(document);
+		} else if (tableName.equals(SearchConstants.PURCHASE_INVOICE_TABLE_NAME)) {
+			return toPurchaseInvoice(document);
+		}
+		throw new SearchException("不支持的表：" + tableName);
+	}
+
 	/**
 	 * 将Document转换为类目对象
 	 * 

search-console/src/main/java/com/uas/search/console/util/FileUtils.java

@@ -0,0 +1,52 @@
+package com.uas.search.console.util;
+
+import java.io.File;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 文件操作
+ * 
+ * @author sunyj
+ * @since 2017年3月22日 上午9:40:41
+ */
+public class FileUtils {
+
+	private static Logger logger = LoggerFactory.getLogger(FileUtils.class);
+
+	/**
+	 * 删除文件夹下所有子文件（不含该文件夹本身）
+	 * 
+	 * @param dir
+	 */
+	public static void deleteSubFiles(File dir) {
+		if (dir == null || !dir.exists() || !dir.isDirectory()) {
+			return;
+		}
+		File[] files = dir.listFiles();
+		for (File file : files) {
+			delete(file);
+		}
+	}
+
+	/**
+	 * 删除文件（子文件及其本身均删除）
+	 * 
+	 * @param file
+	 */
+	public static void delete(File file) {
+		if (file == null || !file.exists()) {
+			return;
+		}
+		if (file.isDirectory()) {
+			File[] files = file.listFiles();
+			for (File f : files) {
+				delete(f);
+			}
+		}
+		file.delete();
+		logger.info("Deleted ..." + file.getPath());
+	}
+
+}

search-console/src/main/java/com/uas/search/console/util/SearchUtils.java

@@ -36,6 +36,7 @@ import com.uas.search.console.model.PurchaseInvoiceSimpleInfo;
 import com.uas.search.console.model.PurchaseSimpleInfo;
 import com.uas.search.console.support.IndexSearcherManager;
 import com.uas.search.exception.SearchException;
+import com.uas.search.model.SPage;
 
 /**
  * 搜索相关的工具类
@@ -281,6 +282,102 @@ public class SearchUtils {
 		return documents;
 	}
 
+	/**
+	 * 获取所有Document
+	 * 
+	 * @param tableName
+	 *            表名
+	 * @param page
+	 *            页码
+	 * @param size
+	 *            页面大小
+	 * @return
+	 */
+	public static SPage<Document> getAllDocuments(String tableName, int page, int size) {
+		IndexSearcher indexSearcher = getIndexSearcher(tableName);
+		SPage<Document> sPage = new SPage<>();
+		try {
+			RegexpQuery query = getRegexpQuery(getIdField(tableName), "");
+			logger.info(query.toString());
+
+			// 分页信息
+			if (page > 0) {
+				sPage.setPage(page);
+			} else {
+				sPage.setPage(1);
+				sPage.setFirst(true);
+			}
+			if (size > 0) {
+				sPage.setSize(size);
+			} else {
+				sPage.setSize(20);
+			}
+
+			TopDocs topDocs;
+			// 如果页码不为1
+			if (sPage.getPage() > 1) {
+				TopDocs previousTopDocs = indexSearcher.search(query, (sPage.getPage() - 1) * sPage.getSize());
+				int totalHits = previousTopDocs.totalHits;
+				ScoreDoc[] previousScoreDocs = previousTopDocs.scoreDocs;
+				if ((sPage.getPage() - 1) * sPage.getSize() >= totalHits) {
+					throw new SearchException("页码过大：元素总数量为" + totalHits);
+				}
+				topDocs = indexSearcher.searchAfter(previousScoreDocs[previousScoreDocs.length - 1], query,
+						sPage.getSize());
+			} else {
+				sPage.setFirst(true);
+				topDocs = indexSearcher.search(query, sPage.getSize());
+			}
+
+			int totalHits = topDocs.totalHits;
+			// 设置总元素个数、页数等信息
+			sPage.setTotalElement(totalHits);
+			int totalPage = (int) Math.ceil(totalHits / (1.0 * sPage.getSize()));
+			sPage.setTotalPage(totalPage);
+			if (totalPage == sPage.getPage()) {
+				sPage.setLast(true);
+			}
+			List<Document> content = new ArrayList<>();
+			for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+				Document document = indexSearcher.doc(scoreDoc.doc);
+				content.add(document);
+			}
+			sPage.setContent(content);
+		} catch (IOException e) {
+			throw new SearchException(e).setDetailedMessage(e);
+		} finally {
+			releaseIndexSearcher(indexSearcher);
+		}
+		return sPage;
+	}
+
+	/**
+	 * 获取各表索引的id字段名
+	 * 
+	 * @param tableName
+	 * @return
+	 */
+	private static String getIdField(String tableName) {
+		if (tableName.equals(SearchConstants.KIND_TABLE_NAME)) {
+			return SearchConstants.KIND_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.BRAND_TABLE_NAME)) {
+			return SearchConstants.BRAND_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.COMPONENT_TABLE_NAME)) {
+			return SearchConstants.COMPONENT_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.GOODS_TABLE_NAME)) {
+			return SearchConstants.GOODS_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.ORDER_TABLE_NAME)) {
+			return SearchConstants.ORDER_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.ORDER_INVOICE_TABLE_NAME)) {
+			return SearchConstants.ORDER_INVOICE_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.PURCHASE_TABLE_NAME)) {
+			return SearchConstants.PURCHASE_ID_FIELD;
+		} else if (tableName.equals(SearchConstants.PURCHASE_INVOICE_TABLE_NAME)) {
+			return SearchConstants.PURCHASE_INVOICE_ID_FIELD;
+		}
+		throw new SearchException("不支持的表：" + tableName);
+	}
+
 	/**
 	 * 获取所有需要建立索引的表名
 	 * 

search-console/src/main/webapp/WEB-INF/views/console.html

@@ -51,6 +51,8 @@
 					<li><a target="_blank">search/brand/12</a></li>
 					<li><a target="_blank">search/component/12</a></li>
 					<li><a target="_blank">search/goods/AAAevvAAHAAI1EIAAB</a></li>
+					<li><a target="_blank">search/allObjects?tableName=product$brand&page=1&size=10</a></li>
+					<li><a target="_blank">search/allObjectsToFiles?tableName=product$brand</a></li>
 				</ol>
 			</ol>
 		</div>