增加下载component数据的接口；索引中增加属性值的rowid

search-console/src/main/java/com/uas/search/console/controller/IndexController.java

@@ -74,49 +74,58 @@ public class IndexController {
 		dcnRegistrationManager.registerDataChangeNotification();
 		return "Started real-time updating for lucene index...";
 	}
-	
+
+	@RequestMapping("/downloadComponentData")
+	@ResponseBody
+	public String downloadComponentData() {
+		return "Components' size: " + indexService.downloadComponentData();
+	}
+
 	@Autowired
 	private ComponentSimpleInfoDao componentDao;
+
 	@RequestMapping("/rowTest/component")
 	@ResponseBody
-	public String testROWIDComponent(){
+	public String testROWIDComponent() {
 		System.out.println(componentDao.findByRowid("AACu7GAAEAAOZvTAAA"));
 		return "created";
 	}
-	
+
 	@Autowired
 	private BrandSimpleInfoDao brandDao;
+
 	@RequestMapping("/rowTest/brand")
 	@ResponseBody
-	public String testROWIDBrand(){
+	public String testROWIDBrand() {
 		System.out.println(brandDao.findByRowid("AACoH2AAEAANmqSAAB"));
 		return "created";
 	}
-	
+
 	@Autowired
 	private KindSimpleInfoDao kindDao;
+
 	@RequestMapping("/rowTest/kind")
 	@ResponseBody
-	public String testROWIDKind(){
+	public String testROWIDKind() {
 		System.out.println(kindDao.findByRowid("AACmtMAAEAAFmG8AAY"));
 		return "created";
 	}
-	
+
 	@RequestMapping("/findAll/kind")
 	@ResponseBody
-	public List<KindSimpleInfo> findAllKinds(){
+	public List<KindSimpleInfo> findAllKinds() {
 		return kindDao.findAll();
 	}
-	
+
 	@RequestMapping("/findAll/brand")
 	@ResponseBody
-	public List<BrandSimpleInfo> findAllBrands(){
+	public List<BrandSimpleInfo> findAllBrands() {
 		return brandDao.findAll();
 	}
-	
+
 	@RequestMapping("/findAll/component")
 	@ResponseBody
-	public List<ComponentSimpleInfo> findAllComponents(){
+	public List<ComponentSimpleInfo> findAllComponents() {
 		return componentDao.findAll();
 	}
 }

search-console/src/main/java/com/uas/search/console/dao/BrandSimpleInfoDao.java

@@ -20,6 +20,7 @@ public interface BrandSimpleInfoDao
 	 * @param id
 	 * @return
 	 */
+	// TODO DCN测试完成后删除
 	// public BrandSimpleInfo findById(Long id);
 
 	/**

search-console/src/main/java/com/uas/search/console/dao/ComponentSimpleInfoDao.java

@@ -24,6 +24,7 @@ public interface ComponentSimpleInfoDao
 	 * @param id
 	 * @return
 	 */
+	// TODO DCN测试完成后删除
 	// public ComponentSimpleInfo findById(Long id);
 
 	/**

search-console/src/main/java/com/uas/search/console/dao/KindSimpleInfoDao.java

@@ -20,6 +20,7 @@ public interface KindSimpleInfoDao
 	 * @param id
 	 * @return
 	 */
+	// TODO DCN测试完成后删除
 	// public KindSimpleInfo findById(Long id);
 
 	/**

search-console/src/main/java/com/uas/search/console/dcn/DCNRegistrationManager.java

@@ -86,7 +86,6 @@ public class DCNRegistrationManager {
 				}
 			}
 		}
-
 	}
 
 	/**

search-console/src/main/java/com/uas/search/console/model/PropertyValue.java

@@ -47,6 +47,8 @@ public class PropertyValue implements Serializable {
 	@Column(name = "pv_value")
 	private String value;
 
+	private String rowid;
+
 	public Long getId() {
 		return id;
 	}
@@ -79,9 +81,17 @@ public class PropertyValue implements Serializable {
 		this.propertyid = propertyid;
 	}
 
+	public String getRowid() {
+		return rowid;
+	}
+
+	public void setRowid(String rowid) {
+		this.rowid = rowid;
+	}
+
 	@Override
 	public String toString() {
-		return "PropertyValue [propertyid=" + propertyid + ", value=" + value + "]";
+		return "PropertyValue [propertyid=" + propertyid + ", value=" + value + ", rowid=" + rowid + "]";
 	}
 
 }

search-console/src/main/java/com/uas/search/console/service/IndexService.java

@@ -39,4 +39,11 @@ public interface IndexService {
 	 */
 	public void delete(Object obj);
 
+	/**
+	 * 下载component数据至本地
+	 * 
+	 * @return component记录的数目
+	 */
+	public Long downloadComponentData();
+
 }

search-console/src/main/java/com/uas/search/console/service/impl/IndexServiceImpl.java

@@ -5,6 +5,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.nio.file.Paths;
 import java.util.Date;
 import java.util.List;
@@ -27,6 +28,7 @@ import org.springframework.util.StringUtils;
 import com.alibaba.fastjson.JSONObject;
 import com.uas.platform.core.model.PageInfo;
 import com.uas.platform.core.model.PageParams;
+import com.uas.search.console.core.util.PathUtils;
 import com.uas.search.console.dao.BrandSimpleInfoDao;
 import com.uas.search.console.dao.ComponentSimpleInfoDao;
 import com.uas.search.console.dao.KindSimpleInfoDao;
@@ -90,7 +92,7 @@ public class IndexServiceImpl implements IndexService {
 		// 清除旧索引
 		File file = new File(SearchConstants.INDEX_DIR);
 		if (file.isDirectory()) {
-			deleteOldIndex(file);
+			clearDir(file);
 		}
 
 		try {
@@ -105,7 +107,7 @@ public class IndexServiceImpl implements IndexService {
 			Long brandTime = new Date().getTime();
 			logger.info("创建品牌索引： " + brandSize + "条，耗时 " + (brandTime - kindTime) + " ms\n");
 
-//			Long componentSize = createComponentIndexesWithFiles();
+			// Long componentSize = createComponentIndexesWithFiles();
 			Long componentSize = createComponentIndexes();
 			Long componentTime = new Date().getTime();
 			logger.info("创建器件索引： " + componentSize + "条，耗时 " + (componentTime - brandTime) + " ms\n");
@@ -208,6 +210,80 @@ public class IndexServiceImpl implements IndexService {
 		return size;
 	}
 
+	// 单个文件存储的最大数据数目
+	public static final int SINGLE_FILE_MAX_SIZE = 100000;
+
+	@Override
+	public Long downloadComponentData() {
+		logger.info("正在下载器件数据...");
+
+		// 清空data文件夹
+		String dataDirPath = PathUtils.getFilePath() + "data" + File.separator;
+		File dataDir = new File(dataDirPath);
+		if (dataDir.isDirectory()) {
+			clearDir(dataDir);
+		}
+		if (!dataDir.exists()) {
+			dataDir.mkdir();
+		}
+
+		Long size = 0L;
+		PageParams params = new PageParams();
+		int page = 1;
+		params.setCount(PAGE_SIZE);
+		params.setPage(page);
+
+		// 分页获取component数据
+		PageInfo info = new PageInfo(params);
+		Page<ComponentSimpleInfo> pageResult = componentDao.findAll(info);
+		long totalElements = pageResult.getTotalElements();
+		logger.info("Number of components: " + totalElements);
+
+		PrintWriter printWriter = null;
+		try {
+			String dataFilePath = dataDirPath + "component_1.json";
+			printWriter = new PrintWriter(dataFilePath);
+			logger.info("Writing data into " + dataFilePath);
+			int fileIndex = 1;
+			int dataCount = 0;
+			while (totalElements > size) {
+				// 一个文件存放SINGLE_FILE_MAX_SIZE条数据，一旦超过，写入新的文件
+				if (dataCount >= SINGLE_FILE_MAX_SIZE) {
+					printWriter.flush();
+					printWriter.close();
+					dataCount = 0;
+					dataFilePath = dataDirPath + "dataDirPath" + (++fileIndex) + ".json";
+					printWriter = new PrintWriter(dataFilePath);
+					logger.info("Writing data into " + dataFilePath);
+				}
+
+				// 获取数据并写入文件
+				List<ComponentSimpleInfo> components = pageResult.getContent();
+				for (ComponentSimpleInfo component : components) {
+					printWriter.println(JSONObject.toJSONString(component));
+				}
+				size += components.size();
+				dataCount += components.size();
+
+				// 打印器件数据的下载进度（百分比）
+				System.out.printf("Downloading components' data...................%.2f%%\n",
+						(size * 100.0 / totalElements));
+
+				page++;
+				params.setPage(page);
+				info = new PageInfo(params);
+				pageResult = componentDao.findAll(info);
+			}
+		} catch (FileNotFoundException e) {
+			e.printStackTrace();
+		} finally {
+			if (printWriter != null) {
+				printWriter.close();
+			}
+		}
+		return size;
+	}
+
 	/**
 	 * 创建器件索引，从数据库取数据
 	 * 
@@ -241,8 +317,8 @@ public class IndexServiceImpl implements IndexService {
 			// 器件索引的创建进度（百分比）
 			double indexProgress = size * 100.0 / totalElements;
 			System.out.printf("Component indexed...................%.2f%%\n", indexProgress);
-			// 每创建5%，提交一次，避免内存耗尽，发生OutOfMemoryError
-			if (indexProgress - recordProgress > 5) {
+			// 每创建1%，提交一次，避免内存耗尽，发生OutOfMemoryError
+			if (indexProgress - recordProgress > 1) {
 				indexWriter.commit();
 				recordProgress = indexProgress;
 			}
@@ -326,6 +402,12 @@ public class IndexServiceImpl implements IndexService {
 		// 属性值加入索引，索引中field的键："pr_"前缀连接属性的id
 		Set<PropertyValue> propertyValues = component.getProperties();
 		for (PropertyValue propertyValue : propertyValues) {
+
+			// 为属性值每行记录的rowid建立索引，用于实时更新索引服务中
+			// （属性值记录被删除时，无法根据DCN返回的属性值记录的rowid得知需要更新哪个器件的索引），
+			// 通过该rowid从本地索引中获得相应器件的id，再根据该id从数据库中获取器件信息，更新本地索引
+			document.add(new StringField(propertyValue.getRowid(), "1", Store.YES));
+
 			if (!StringUtils.isEmpty(propertyValue.getValue())) {
 				String fieldKey = SearchConstants.COMPONENT_PROPERTY_PREFIX + propertyValue.getPropertyid();
 				document.add(new StringField(fieldKey, propertyValue.getValue(), Store.YES));
@@ -339,24 +421,23 @@ public class IndexServiceImpl implements IndexService {
 	}
 
 	/**
-	 * 删除原有索引文件
+	 * 清空文件夹
 	 * 
-	 * @param file
+	 * @param dir
 	 */
-	private void deleteOldIndex(File file) {
-		if (file == null) {
+	private void clearDir(File dir) {
+		if (dir == null) {
 			return;
 		}
-		if (file.isDirectory()) {
-			File[] files = file.listFiles();
+		if (dir.isDirectory()) {
+			File[] files = dir.listFiles();
 			for (File f : files) {
-				deleteOldIndex(f);
+				clearDir(f);
 			}
 		}
-
-		file.delete();
+		dir.delete();
 		try {
-			logger.info("deleted " + file.getCanonicalPath());
+			logger.info("Deleted " + dir.getCanonicalPath());
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
@@ -396,7 +477,7 @@ public class IndexServiceImpl implements IndexService {
 		} finally {
 			indexWriterManager.release();
 		}
-		logger.info("saved object... " + obj + "\n");
+		logger.info("Saved object... " + obj + "\n");
 	}
 
 	@Override
@@ -436,7 +517,7 @@ public class IndexServiceImpl implements IndexService {
 		} finally {
 			indexWriterManager.release();
 		}
-		logger.info("updated object... " + obj + "\n");
+		logger.info("Updated object... " + obj + "\n");
 	}
 
 	@Override
@@ -466,6 +547,6 @@ public class IndexServiceImpl implements IndexService {
 		} finally {
 			indexWriterManager.release();
 		}
-		logger.info("deleted object... " + obj + "\n");
+		logger.info("Deleted object... " + obj + "\n");
 	}
 }