创建索引时可选择器件数据来自数据库或本地文件

search-console/src/main/java/com/uas/search/console/controller/IndexController.java

@@ -2,6 +2,7 @@ package com.uas.search.console.controller;
 
 import java.util.List;
 
+import org.apache.log4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -11,7 +12,6 @@ import com.uas.search.console.dao.BrandSimpleInfoDao;
 import com.uas.search.console.dao.ComponentSimpleInfoDao;
 import com.uas.search.console.dao.KindSimpleInfoDao;
 import com.uas.search.console.dcn.DCNRegistrationManager;
-import com.uas.search.console.jms.AQListener;
 import com.uas.search.console.model.BrandSimpleInfo;
 import com.uas.search.console.model.ComponentSimpleInfo;
 import com.uas.search.console.model.KindSimpleInfo;
@@ -30,37 +30,22 @@ public class IndexController {
 	@Autowired
 	private IndexService indexService;
 
-	@Autowired
-	private AQListener aqListener;
-
 	@Autowired
 	private DCNRegistrationManager dcnRegistrationManager;
 
+	private Logger logger = Logger.getLogger(IndexController.class);
+
 	/**
-	 * 初始化创建索引文件
+	 * 创建索引
 	 * 
+	 * @param componentDataSource
+	 *            器件数据的来源，为1表示从数据库获取，为null或其他值表示从本地文件获取
 	 * @return 所用时间 ms
 	 */
 	@RequestMapping("/create")
 	@ResponseBody
-	public String initIndexes() {
-		return "Indexes created success in " + indexService.createIndexs() + " ms.";
-	}
-
-	/**
-	 * 实时更新索引
-	 * 
-	 * @return
-	 */
-	@RequestMapping("/listen")
-	@ResponseBody
-	public String listen() {
-		new Thread() {
-			public void run() {
-				aqListener.execute();
-			};
-		}.start();
-		return "Listen...";
+	public String createIndexes(Integer componentDataSource) {
+		return "Indexes created success in " + indexService.createIndexes(componentDataSource) + " ms.";
 	}
 
 	/**
@@ -78,7 +63,9 @@ public class IndexController {
 	@RequestMapping("/downloadComponentData")
 	@ResponseBody
 	public String downloadComponentData() {
-		return "Components' size: " + indexService.downloadComponentData();
+		Long size = indexService.downloadComponentData();
+		logger.info("Downloading finished, components' size " + size);
+		return "Components' size: " + size;
 	}
 
 	@Autowired

search-console/src/main/java/com/uas/search/console/service/IndexService.java

@@ -9,11 +9,20 @@ package com.uas.search.console.service;
 public interface IndexService {
 
 	/**
-	 * 初始化时，从数据库中得到全部类目、品牌和器件对象，写进索引中
+	 * 创建索引
 	 * 
+	 * @param componentDataSource
+	 *            器件数据的来源，为1表示从数据库获取，为null或其他值表示从本地文件获取
 	 * @return 创建的索引花费总时间 ms
 	 */
-	public Long createIndexs();
+	public Long createIndexes(Integer componentDataSource);
+
+	/**
+	 * 下载component数据至本地
+	 * 
+	 * @return component记录的数目
+	 */
+	public Long downloadComponentData();
 
 	/**
 	 * 将新对象添加在lucene索引中
@@ -39,11 +48,4 @@ public interface IndexService {
 	 */
 	public void delete(Object obj);
 
-	/**
-	 * 下载component数据至本地
-	 * 
-	 * @return component记录的数目
-	 */
-	public Long downloadComponentData();
-
 }

search-console/src/main/java/com/uas/search/console/service/impl/IndexServiceImpl.java

@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.io.PrintWriter;
 import java.nio.file.Paths;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -28,7 +29,6 @@ import org.springframework.util.StringUtils;
 import com.alibaba.fastjson.JSONObject;
 import com.uas.platform.core.model.PageInfo;
 import com.uas.platform.core.model.PageParams;
-import com.uas.search.console.core.util.PathUtils;
 import com.uas.search.console.dao.BrandSimpleInfoDao;
 import com.uas.search.console.dao.ComponentSimpleInfoDao;
 import com.uas.search.console.dao.KindSimpleInfoDao;
@@ -38,7 +38,6 @@ import com.uas.search.console.model.KindSimpleInfo;
 import com.uas.search.console.model.PropertyValue;
 import com.uas.search.console.service.IndexService;
 import com.uas.search.console.support.IndexWriterManager;
-import com.uas.search.console.util.MergeComponentData;
 import com.uas.search.console.util.SearchConstants;
 
 /**
@@ -65,9 +64,13 @@ public class IndexServiceImpl implements IndexService {
 
 	private FSDirectory directory;
 
+	private Logger logger = Logger.getLogger(IndexServiceImpl.class);
+
+	// 分页获取component数据时的页的大小
 	private static final int PAGE_SIZE = 1000;
 
-	private Logger logger = Logger.getLogger(IndexServiceImpl.class);
+	// 单个文件存储的最大数据数目
+	public static final int SINGLE_FILE_MAX_SIZE = 100000;
 
 	public IndexServiceImpl() {
 		try {
@@ -79,7 +82,7 @@ public class IndexServiceImpl implements IndexService {
 	}
 
 	@Override
-	public Long createIndexs() {
+	public Long createIndexes(Integer componentDataSource) {
 		try {
 			if (IndexWriter.isLocked(directory)) {
 				logger.warn("已有线程正在创建索引！");
@@ -107,8 +110,12 @@ public class IndexServiceImpl implements IndexService {
 			Long brandTime = new Date().getTime();
 			logger.info("创建品牌索引： " + brandSize + "条，耗时 " + (brandTime - kindTime) + " ms\n");
 
-			// Long componentSize = createComponentIndexesWithFiles();
-			Long componentSize = createComponentIndexes();
+			Long componentSize = 0L;
+			if (componentDataSource != null && componentDataSource == 1) {
+				componentSize = createComponentIndexesFromDatabase();
+			} else {
+				componentSize = createComponentIndexesFromFiles();
+			}
 			Long componentTime = new Date().getTime();
 			logger.info("创建器件索引： " + componentSize + "条，耗时 " + (componentTime - brandTime) + " ms\n");
 
@@ -170,18 +177,23 @@ public class IndexServiceImpl implements IndexService {
 		return (long) brands.size();
 	}
 
-	public Long createComponentIndexesWithFiles() {
+	/**
+	 * 从本地文件获取器件数据
+	 * 
+	 * @return
+	 */
+	private Long createComponentIndexesFromFiles() {
 		logger.info("正在创建器件索引...");
 		Long size = 0L;
 		try {
 			// 从本地路径读取器件数据
-			File[] files = new File(SearchConstants.COMPONENT_WITH_PROPERTY_DIR).listFiles();
+			File[] files = new File(SearchConstants.COMPONENT_DATA_DIR).listFiles();
 			if (files == null || files.length == 0) {
 				logger.info("创建器件索引失败，原因：器件数据文件不存在！");
 				return 0L;
 			}
 			// 将要创建的索引总数目约为：文件数目*单个文件的行数
-			long totalSize = files.length * MergeComponentData.SINGLE_FILE_MAX_SIZE;
+			long totalSize = files.length * SINGLE_FILE_MAX_SIZE;
 			for (File file : files) {
 				logger.info("读取文件: " + file.getName());
 				BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
@@ -193,7 +205,8 @@ public class IndexServiceImpl implements IndexService {
 						size++;
 						// 每创建10000条，打印一次进度
 						if (size % 10000 == 0) {
-							System.out.printf("Component indexed...................%.2f%%\n", size * 100.0 / totalSize);
+							logger.info(
+									String.format("Component indexed...............%.2f%%", size * 100.0 / totalSize));
 						}
 						indexWriter.addDocument(document);
 					}
@@ -210,16 +223,12 @@ public class IndexServiceImpl implements IndexService {
 		return size;
 	}
 
-	// 单个文件存储的最大数据数目
-	public static final int SINGLE_FILE_MAX_SIZE = 100000;
-
 	@Override
 	public Long downloadComponentData() {
 		logger.info("正在下载器件数据...");
 
 		// 清空data文件夹
-		String dataDirPath = PathUtils.getFilePath() + "data" + File.separator;
-		File dataDir = new File(dataDirPath);
+		File dataDir = new File(SearchConstants.COMPONENT_DATA_DIR);
 		if (dataDir.isDirectory()) {
 			clearDir(dataDir);
 		}
@@ -241,7 +250,7 @@ public class IndexServiceImpl implements IndexService {
 
 		PrintWriter printWriter = null;
 		try {
-			String dataFilePath = dataDirPath + "component_1.json";
+			String dataFilePath = SearchConstants.COMPONENT_DATA_DIR + "component_1.json";
 			printWriter = new PrintWriter(dataFilePath);
 			logger.info("Writing data into " + dataFilePath);
 			int fileIndex = 1;
@@ -252,7 +261,7 @@ public class IndexServiceImpl implements IndexService {
 					printWriter.flush();
 					printWriter.close();
 					dataCount = 0;
-					dataFilePath = dataDirPath + "dataDirPath" + (++fileIndex) + ".json";
+					dataFilePath = SearchConstants.COMPONENT_DATA_DIR + "component_" + (++fileIndex) + ".json";
 					printWriter = new PrintWriter(dataFilePath);
 					logger.info("Writing data into " + dataFilePath);
 				}
@@ -260,14 +269,25 @@ public class IndexServiceImpl implements IndexService {
 				// 获取数据并写入文件
 				List<ComponentSimpleInfo> components = pageResult.getContent();
 				for (ComponentSimpleInfo component : components) {
+					// 属性值的value为空，不写入索引
+					Set<PropertyValue> properties = component.getProperties();
+					Set<PropertyValue> removingProperties = new HashSet<>();
+					for (PropertyValue property : properties) {
+						if (StringUtils.isEmpty(property.getValue())) {
+							removingProperties.add(property);
+						}
+					}
+					properties.removeAll(removingProperties);
+					component.setProperties(properties);
 					printWriter.println(JSONObject.toJSONString(component));
 				}
 				size += components.size();
 				dataCount += components.size();
 
 				// 打印器件数据的下载进度（百分比）
-				System.out.printf("Downloading components' data...................%.2f%%\n",
-						(size * 100.0 / totalElements));
+				if (size % 10000 == 0) {
+					logger.info(String.format("Downloading...............%.2f%%", (size * 100.0 / totalElements)));
+				}
 
 				page++;
 				params.setPage(page);
@@ -290,7 +310,7 @@ public class IndexServiceImpl implements IndexService {
 	 * @return 写入的器件索引数
 	 * @throws IOException
 	 */
-	public Long createComponentIndexes() throws IOException {
+	private Long createComponentIndexesFromDatabase() throws IOException {
 		logger.info("正在创建器件索引...");
 		Long size = 0L;
 		PageParams params = new PageParams();
@@ -316,9 +336,9 @@ public class IndexServiceImpl implements IndexService {
 
 			// 器件索引的创建进度（百分比）
 			double indexProgress = size * 100.0 / totalElements;
-			System.out.printf("Component indexed...................%.2f%%\n", indexProgress);
-			// 每创建1%，提交一次，避免内存耗尽，发生OutOfMemoryError
-			if (indexProgress - recordProgress > 1) {
+			// 每创建0.3%，提交一次，避免内存耗尽，发生OutOfMemoryError
+			if (indexProgress - recordProgress > 0.3) {
+				logger.info(String.format("Component indexed...............%.2f%%", indexProgress));
 				indexWriter.commit();
 				recordProgress = indexProgress;
 			}
@@ -446,9 +466,9 @@ public class IndexServiceImpl implements IndexService {
 	@Override
 	public void save(Object obj) {
 		if (obj == null) {
+			logger.error("实时更新对象为空");
 			return;
 		}
-
 		try {
 			indexWriter = indexWriterManager.get();
 			if (obj instanceof KindSimpleInfo) {
@@ -483,6 +503,7 @@ public class IndexServiceImpl implements IndexService {
 	@Override
 	public void update(Object obj) {
 		if (obj == null) {
+			logger.error("实时更新对象为空");
 			return;
 		}
 		try {
@@ -523,6 +544,7 @@ public class IndexServiceImpl implements IndexService {
 	@Override
 	public void delete(Object obj) {
 		if (obj == null) {
+			logger.error("实时更新对象为空");
 			return;
 		}
 		try {