创建器件索引时改从本地获取数据

search-console/src/main/java/com/uas/search/console/controller/SearchController.java

@@ -1,5 +1,6 @@
 package com.uas.search.console.controller;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -9,6 +10,8 @@ import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
 
+import com.uas.search.console.dao.ComponentSimpleInfoDao;
+import com.uas.search.console.model.ComponentSimpleInfo;
 import com.uas.search.model.PageParams;
 import com.uas.search.service.SearchService;
 
@@ -72,8 +75,9 @@ public class SearchController {
 	public List<Map<String, Object>> searchBrand(String keyword) {
 		return searchService.getBrands(keyword);
 	}
+
 	/**
-	 * 搜索产品
+	 * 搜索产品id
 	 * 
 	 * @param keyword
 	 * @return
@@ -84,6 +88,32 @@ public class SearchController {
 		return searchService.getComponentIds(keyword, new PageParams());
 	}
 
+	/**
+	 * 搜索产品
+	 * 
+	 * @param keyword
+	 * @return
+	 */
+	@Autowired
+	private ComponentSimpleInfoDao componentDao;
+
+	@RequestMapping("/components")
+	@ResponseBody
+	public List<ComponentSimpleInfo> searchComponents(String keyword) {
+		PageParams pageParams = new PageParams();
+		Map<String, Object> filters = new HashMap<>();
+		filters.put("pr_329", "HRC");
+		pageParams.setFilters(filters);
+		@SuppressWarnings("unchecked")
+		List<Long> ids = (List<Long>) searchService.getComponentIds(keyword, pageParams).get("components");
+		Long[] idsLong = new Long[ids.size()];
+		int i = 0;
+		for (Long id : ids) {
+			idsLong[i++] = id;
+		}
+		return componentDao.findByIdsInOrder(idsLong);
+	}
+
 	/**
 	 * 统计器件的类目id
 	 * 

search-console/src/main/java/com/uas/search/console/dao/ComponentSimpleInfoDao.java

@@ -4,6 +4,8 @@ import java.util.List;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
 
 import com.uas.search.console.model.ComponentSimpleInfo;
@@ -39,4 +41,13 @@ public interface ComponentSimpleInfoDao
 	 * @return
 	 */
 	public List<ComponentSimpleInfo> findByBrandid(Long brandid);
+
+	/**
+	 * 按id获取器件，并且按照给定的id的顺序
+	 * 
+	 * @param ids
+	 * @return
+	 */
+	@Query("select c from ComponentSimpleInfo c where c.id in (:ids) order by instr(':ids', c.id)")
+	public List<ComponentSimpleInfo> findByIdsInOrder(@Param("ids") Long[] ids);
 }

search-console/src/main/java/com/uas/search/console/model/PropertyValue.java

@@ -111,4 +111,9 @@ public class PropertyValue implements Serializable {
 		this.propertyid = propertyid;
 	}
 
+	@Override
+	public String toString() {
+		return "PropertyValue [propertyid=" + propertyid + ", value=" + value + "]";
+	}
+
 }

search-console/src/main/java/com/uas/search/console/service/impl/IndexServiceImpl.java

@@ -1,6 +1,9 @@
 package com.uas.search.console.service.impl;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.Date;
@@ -20,6 +23,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
+import com.alibaba.fastjson.JSONObject;
 import com.uas.platform.core.model.PageInfo;
 import com.uas.platform.core.model.PageParams;
 import com.uas.search.console.dao.BrandSimpleInfoDao;
@@ -31,6 +35,7 @@ import com.uas.search.console.model.KindSimpleInfo;
 import com.uas.search.console.model.PropertyValue;
 import com.uas.search.console.service.IndexService;
 import com.uas.search.console.support.IndexWriterManager;
+import com.uas.search.console.util.MergeComponentData;
 import com.uas.search.console.util.SearchConstants;
 
 /**
@@ -97,7 +102,8 @@ public class IndexServiceImpl implements IndexService {
 			Long brandTime = new Date().getTime();
 			System.out.println("创建品牌索引： " + brandSize + "条，耗时 " + (brandTime - kindTime) + " ms\n");
 
-			Long componentSize = createComponentIndexs();
+			// Long componentSize = createComponentIndexs();
+			Long componentSize = createComponentIndexesWithFiles();
 			Long componentTime = new Date().getTime();
 			System.out.println("创建器件索引： " + componentSize + "条，耗时 " + (componentTime - brandTime) + " ms\n");
 
@@ -159,6 +165,46 @@ public class IndexServiceImpl implements IndexService {
 		return (long) brands.size();
 	}
 
+	private Long createComponentIndexesWithFiles() {
+		System.out.println("正在创建器件索引...");
+		Long size = 0L;
+		try {
+			// 从本地路径读取器件数据
+			File[] files = new File(SearchConstants.COMPONENT_WITH_PROPERTY_DIR).listFiles();
+			// if(files.length == 0){
+			// System.out.println("创建器件索引失败，原因：器件数据文件不存在！");
+			// return 0L;
+			// }
+			// 将要创建的索引总数目约为：文件数目*单个文件的行数
+			long totalSize = files.length * MergeComponentData.SIMGLE_FILE_MAX_SIZE;
+			for (File file : files) {
+				System.out.println("创建器件索引自文件: " + file.getName());
+				BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
+				String line = null;
+				while (!StringUtils.isEmpty(line = bufferedReader.readLine())) {
+					ComponentSimpleInfo component = JSONObject.parseObject(line, ComponentSimpleInfo.class);
+					Document document = toDocument(component);
+					if (document != null) {
+						size++;
+						// 每创建10000条，打印一次进度
+						if (size % 10000 == 0) {
+							System.out.printf("Component indexed...................%.2f%%\n", size * 100.0 / totalSize);
+						}
+						indexWriter.addDocument(document);
+					}
+				}
+				indexWriter.commit();
+				bufferedReader.close();
+			}
+		} catch (FileNotFoundException e) {
+			System.out.println("创建器件索引失败，原因：器件数据文件不存在！");
+			return 0L;
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		return size;
+	}
+
 	/**
 	 * 创建器件索引
 	 * 
@@ -177,6 +223,7 @@ public class IndexServiceImpl implements IndexService {
 		Page<ComponentSimpleInfo> pageResult = componentDao.findAll(info);
 		long totalElements = pageResult.getTotalElements();
 		System.out.println("Number of components: " + totalElements);
+		// 用于记录上次提交索引时的创建进度
 		double recordProgress = 0;
 		while (totalElements > size) {
 			List<ComponentSimpleInfo> components = pageResult.getContent();
@@ -191,8 +238,8 @@ public class IndexServiceImpl implements IndexService {
 			// 器件索引的创建进度（百分比）
 			double indexProgress = size * 100.0 / totalElements;
 			System.out.printf("Component indexed...................%.2f%%\n", indexProgress);
-			// 每创建10%，提交一次，避免内存耗尽，发生OutOfMemoryError
-			if (indexProgress - recordProgress > 10) {
+			// 每创建5%，提交一次，避免内存耗尽，发生OutOfMemoryError
+			if (indexProgress - recordProgress > 5) {
 				indexWriter.commit();
 				recordProgress = indexProgress;
 			}
@@ -265,14 +312,11 @@ public class IndexServiceImpl implements IndexService {
 		document.add(new StringField(SearchConstants.COMPONENT_BRANDID_FIELD, String.valueOf(component.getBrandid()),
 				Store.YES));
 
-		// 属性值加入索引，索引中field的键："pr_"前缀连接属性id
+		// 属性值加入索引，索引中field的键："pr_"前缀连接属性的id
 		Set<PropertyValue> propertyValues = component.getProperties();
 		for (PropertyValue propertyValue : propertyValues) {
-			if (!StringUtils.isEmpty(propertyValue.getValue())) {
-				String fieldKey = new StringBuilder(SearchConstants.COMPONENT_PROPERTY_PREFIX)
-						.append(propertyValue.getId()).toString();
-				document.add(new StringField(fieldKey, propertyValue.getValue(), Store.YES));
-			}
+			String fieldKey = SearchConstants.COMPONENT_PROPERTY_PREFIX + propertyValue.getPropertyid();
+			document.add(new StringField(fieldKey, propertyValue.getValue(), Store.YES));
 		}
 		return document;
 	}

search-console/src/main/java/com/uas/search/console/service/impl/SearchServiceImpl.java

@@ -251,15 +251,16 @@ public class SearchServiceImpl implements SearchService {
 			Map<String, Object> filters = page.getFilters();
 			if (!StringUtils.isEmpty(filters)) {
 				// 筛选类目
+				// 若不存在kindId，则String.valueOf(null)会赋值"null"，所以要判断是否等于字符串"null"
 				String kindId = String.valueOf(filters.remove("kindId"));
-				if (!StringUtils.isEmpty(kindId)) {
+				if (!StringUtils.isEmpty(kindId) && !kindId.equals("null")) {
 					TermQuery kindQuery = new TermQuery(new Term(SearchConstants.COMPONENT_KINDID_FIELD, kindId));
 					booleanQuery.add(kindQuery, BooleanClause.Occur.MUST);
 				}
 
 				// 筛选品牌
 				String brandId = String.valueOf(filters.remove("brandId"));
-				if (!StringUtils.isEmpty(brandId)) {
+				if (!StringUtils.isEmpty(brandId) && !brandId.equals("null")) {
 					TermQuery brandQuery = new TermQuery(new Term(SearchConstants.COMPONENT_BRANDID_FIELD, brandId));
 					booleanQuery.add(brandQuery, BooleanClause.Occur.MUST);
 				}
@@ -267,7 +268,7 @@ public class SearchServiceImpl implements SearchService {
 				// 属性过滤
 				for (String key : filters.keySet()) {
 					if (!key.startsWith(SearchConstants.COMPONENT_PROPERTY_PREFIX)) {
-						key = new StringBuilder(SearchConstants.COMPONENT_PROPERTY_PREFIX).append(key).toString();
+						key = SearchConstants.COMPONENT_PROPERTY_PREFIX + key;
 					}
 					String value = String.valueOf(filters.get(key));
 					if (!StringUtils.isEmpty(value)) {

search-console/src/main/java/com/uas/search/console/util/MergeComponentData.java

@@ -0,0 +1,159 @@
+package com.uas.search.console.util;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.springframework.util.StringUtils;
+
+import com.alibaba.fastjson.JSONObject;
+import com.uas.search.console.core.util.FastjsonUtils;
+import com.uas.search.console.model.ComponentSimpleInfo;
+import com.uas.search.console.model.PropertyValue;
+
+/**
+ * 用于对从数据库导出的器件和属性值两个文本文件进行处理，将每个器件所拥有的属性值提取出来，之后每个器件作为一行，以json的形式写入多个文件中
+ * 
+ * @author sunyj
+ * @since 2016年8月8日 下午9:00:31
+ */
+public class MergeComponentData {
+
+	// 单个文件存储的最大数据数目
+	public static final int SIMGLE_FILE_MAX_SIZE = 100000;
+
+	// 器件和属性值两个文本文件的路径
+	private static final String DATA_DIR = "C:\\Users\\sunyj-pc\\Desktop\\data";
+
+	/**
+	 * 将从本地文件读取的一行字符串数据解析为器件对象
+	 * 
+	 * @param data
+	 *            格式为"CMP_ID" "CMP_BRID" "CMP_CODE" "CMP_KIID"，如1 149
+	 *            "10120-5212PC" 295
+	 * @return
+	 */
+	private static ComponentSimpleInfo parseComponent(String data) {
+		ComponentSimpleInfo component = new ComponentSimpleInfo();
+		String[] strs = data.split("\t");
+		component.setId(Long.parseLong(strs[0]));
+		component.setBrandid(Long.parseLong(strs[1]));
+		component.setCode(strs[2].substring(1, strs[2].length() - 1));
+		component.setKindid(Long.parseLong(strs[3]));
+		return component;
+	}
+
+	/**
+	 * 将从本地文件读取的一行字符串数据解析为属性值对象
+	 * 
+	 * @param data
+	 *            格式为"PV_COMPONENTID" "PV_PROPERTYID" "PV_VALUE"，如1 394
+	 *            "Copper Alloy"
+	 * @return
+	 */
+	private static PropertyValue parsePropertyValue(String data) {
+		PropertyValue propertyValue = new PropertyValue();
+		String[] strs = data.split("\t");
+		propertyValue.setComponentid(Long.parseLong(strs[0]));
+		propertyValue.setPropertyid(Long.parseLong(strs[1]));
+		propertyValue.setValue(strs[2].substring(1, strs[2].length() - 1));
+		return propertyValue;
+	}
+
+	public static void merge() {
+		BufferedReader componentReader = null;
+		BufferedReader propertyValueReader = null;
+		PrintWriter printWriter = null;
+		// 分成多个文件存储
+		int fileIndex = 1;
+		try {
+			componentReader = new BufferedReader(new FileReader(DATA_DIR + "\\Component.txt"));
+			propertyValueReader = new BufferedReader(new FileReader(DATA_DIR + "\\PropertyValue.txt"));
+			printWriter = new PrintWriter(DATA_DIR + "\\merged\\ComponentWithProperty_" + fileIndex + ".txt");
+			// 第一行内容为列名，不做解析
+			componentReader.readLine();
+			propertyValueReader.readLine();
+			String componentLine = null;
+			String propertyValueLine = null;
+			PropertyValue propertyValue = null;
+			int dataCount = 0;
+			while (!StringUtils.isEmpty(componentLine = componentReader.readLine())) {
+				dataCount++;
+				System.out.println(dataCount);
+				// 一个文件存放100000条数据，一旦超过，写入新的文件
+				if (dataCount > SIMGLE_FILE_MAX_SIZE) {
+					System.out.println("--------------------------------new file");
+					dataCount = 1;
+					printWriter.flush();
+					printWriter.close();
+					fileIndex++;
+					printWriter = new PrintWriter(DATA_DIR + "\\merged\\ComponentWithProperty_" + fileIndex + ".txt");
+				}
+
+				ComponentSimpleInfo component = parseComponent(componentLine);
+				Set<PropertyValue> properties = new HashSet<>();
+				if (propertyValue == null && !StringUtils.isEmpty(propertyValueLine = propertyValueReader.readLine())) {
+					propertyValue = parsePropertyValue(propertyValueLine);
+				}
+				while (propertyValue.getComponentid().equals(component.getId())) {
+					propertyValue.setComponentid(null);
+					properties.add(propertyValue);
+					// 读到文件最后一行
+					if (StringUtils.isEmpty(propertyValueLine = propertyValueReader.readLine())) {
+						break;
+					}
+					propertyValue = parsePropertyValue(propertyValueLine);
+				}
+				component.setProperties(properties);
+				System.out.println(component);
+				printWriter.println(FastjsonUtils.toJson(component));
+			}
+			printWriter.flush();
+		} catch (FileNotFoundException e) {
+			System.out.println("File not found!");
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			try {
+				componentReader.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			try {
+				propertyValueReader.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			printWriter.close();
+		}
+	}
+
+	private static void readData() {
+		BufferedReader br = null;
+		try {
+			br = new BufferedReader(new FileReader(DATA_DIR + "\\merged\\ComponentWithProperty_1.txt"));
+			String line = br.readLine();
+			System.out.println(JSONObject.parseObject(line, ComponentSimpleInfo.class));
+		} catch (FileNotFoundException e) {
+			System.out.println("File not found!");
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			try {
+				br.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	public static void main(String[] args) {
+		merge();
+		// readData();
+	}
+
+}

search-console/src/main/java/com/uas/search/console/util/SearchConstants.java

@@ -56,6 +56,11 @@ public class SearchConstants {
 	 */
 	public static final String INDEX_DIR = PathUtils.getFilePath() + "indexes";
 
+	/**
+	 * 存放带有属性的器件数据的文件，用于创建索引
+	 */
+	public static final String COMPONENT_WITH_PROPERTY_DIR = PathUtils.getFilePath() + "components";
+
 	/**
 	 * 各索引字段的字符标示
 	 */