联想词排序、去重

search-console/src/main/java/com/uas/search/console/dao/ComponentSimpleInfoDao.java

@@ -32,7 +32,7 @@ public interface ComponentSimpleInfoDao
 	 * @param kindid
 	 * @return
 	 */
-	public List<ComponentSimpleInfo> findByKindid(Long kindid);
+	public List<ComponentSimpleInfo> findByKindId(Long kindId);
 
 	/**
 	 * 根据品牌brandid获取器件
@@ -40,7 +40,7 @@ public interface ComponentSimpleInfoDao
 	 * @param brandid
 	 * @return
 	 */
-	public List<ComponentSimpleInfo> findByBrandid(Long brandid);
+	public List<ComponentSimpleInfo> findByBrandId(Long brandId);
 
 	/**
 	 * 按id获取器件，并且按照给定的id的顺序

search-console/src/main/java/com/uas/search/console/model/ComponentSimpleInfo.java

@@ -9,6 +9,7 @@ import javax.persistence.Entity;
 import javax.persistence.FetchType;
 import javax.persistence.Id;
 import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
 import javax.persistence.OneToMany;
 import javax.persistence.OrderBy;
 import javax.persistence.Table;
@@ -48,16 +49,18 @@ public class ComponentSimpleInfo implements Serializable {
 	private String code;
 
 	/**
-	 * 类目id
+	 * 类目
 	 */
-	@Column(name = "cmp_kiid")
-	private Long kindid;
+	@ManyToOne
+	@JoinColumn(name = "cmp_kiid")
+	private KindSimpleInfo kind;
 
 	/**
-	 * 品牌id
+	 * 品牌
 	 */
-	@Column(name = "cmp_brid")
-	private Long brandid;
+	@ManyToOne
+	@JoinColumn(name = "cmp_brid")
+	private BrandSimpleInfo brand;
 
 	/**
 	 * 以下为器件的库存交易属性，由器件对应的上架商品发生变化时，更新反应到器件
@@ -119,20 +122,20 @@ public class ComponentSimpleInfo implements Serializable {
 		this.code = code;
 	}
 
-	public Long getKindid() {
-		return kindid;
+	public KindSimpleInfo getKind() {
+		return kind;
 	}
 
-	public void setKindid(Long kindid) {
-		this.kindid = kindid;
+	public void setKind(KindSimpleInfo kind) {
+		this.kind = kind;
 	}
 
-	public Long getBrandid() {
-		return brandid;
+	public BrandSimpleInfo getBrand() {
+		return brand;
 	}
 
-	public void setBrandid(Long brandid) {
-		this.brandid = brandid;
+	public void setBrand(BrandSimpleInfo brand) {
+		this.brand = brand;
 	}
 
 	public Double getReserve() {
@@ -177,9 +180,9 @@ public class ComponentSimpleInfo implements Serializable {
 
 	@Override
 	public String toString() {
-		return "ComponentSimpleInfo [id=" + id + ", uuid=" + uuid + ", code=" + code + ", kindid=" + kindid
-				+ ", brandid=" + brandid + ", reserve=" + reserve + ", sampleQty=" + sampleQty + ", originalQty="
-				+ originalQty + ", inactionStockQty=" + inactionStockQty + ", properties=" + properties + "]";
+		return "ComponentSimpleInfo [id=" + id + ", uuid=" + uuid + ", code=" + code + ", kind=" + kind + ", brand="
+				+ brand + ", reserve=" + reserve + ", sampleQty=" + sampleQty + ", originalQty=" + originalQty
+				+ ", inactionStockQty=" + inactionStockQty + ", properties=" + properties + "]";
 	}
 
 }

search-console/src/main/java/com/uas/search/console/service/impl/SearchServiceImpl.java

@@ -11,14 +11,18 @@ import java.util.Set;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.sandbox.queries.DuplicateFilter;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FilteredQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.NumericRangeQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.slf4j.Logger;
@@ -33,6 +37,7 @@ import com.uas.search.console.model.BrandSimpleInfo;
 import com.uas.search.console.model.ComponentSimpleInfo;
 import com.uas.search.console.model.KindSimpleInfo;
 import com.uas.search.console.service.InnerSearchService;
+import com.uas.search.console.support.SimilarValuesFieldComparatorSource;
 import com.uas.search.console.util.DocumentToObjectUtils;
 import com.uas.search.console.util.SearchConstants;
 import com.uas.search.console.util.SearchUtils;
@@ -722,7 +727,8 @@ public class SearchServiceImpl implements SearchService, InnerSearchService {
 				query = new PrefixQuery(new Term(field, keyword));
 			}
 			logger.info(query.toString());
-			TopDocs hits = indexSearcher.search(query, SIMILAR_NUM);
+			Sort sort = new Sort(new SortField(field, new SimilarValuesFieldComparatorSource()));
+			TopDocs hits = indexSearcher.search(new FilteredQuery(query, new DuplicateFilter(field)), SIMILAR_NUM, sort);
 			ScoreDoc[] scoreDocs = hits.scoreDocs;
 			for (ScoreDoc scoreDoc : scoreDocs) {
 				Set<String> fieldsToLoad = new HashSet<>();

search-console/src/main/java/com/uas/search/console/support/SimilarValuesFieldComparator.java

@@ -0,0 +1,80 @@
+package com.uas.search.console.support;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SimpleFieldComparator;
+
+public class SimilarValuesFieldComparator extends SimpleFieldComparator<String> {
+
+	private String fieldname;
+	private String[] values;
+	private BinaryDocValues binaryDocValues;
+
+	public SimilarValuesFieldComparator(String fieldname, int numHits) {
+		this.fieldname = fieldname;
+		this.values = new String[numHits];
+	}
+
+	@Override
+	public void setBottom(int slot) {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public int compareBottom(int doc) throws IOException {
+		// TODO Auto-generated method stub
+		return 0;
+	}
+
+	@Override
+	public int compareTop(int doc) throws IOException {
+		// TODO Auto-generated method stub
+		return 0;
+	}
+
+	@Override
+	public void copy(int slot, int doc) throws IOException {
+		values[slot] = binaryDocValues.get(doc).utf8ToString();
+		System.out.println(slot + "\t" + values[slot]);
+	}
+
+	@Override
+	protected void doSetNextReader(LeafReaderContext context) throws IOException {
+		binaryDocValues = context.reader().getBinaryDocValues(fieldname);
+	}
+
+	@Override
+	public int compare(int slot1, int slot2) {
+		String value1 = values[slot1];
+		String value2 = values[slot2];
+		if (value1 == null) {
+			return 1;
+		} else if (value2 == null) {
+			return -1;
+		}
+
+		if (value1.length() > value2.length()) {
+			return 1;
+		} else if (value1.length() < value2.length()) {
+			return -1;
+		}
+
+		// TODO Auto-generated method stub
+		return 0;
+	}
+
+	@Override
+	public void setTopValue(String value) {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public String value(int slot) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+}

search-console/src/main/java/com/uas/search/console/support/SimilarValuesFieldComparatorSource.java

@@ -0,0 +1,14 @@
+package com.uas.search.console.support;
+
+import java.io.IOException;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+
+public class SimilarValuesFieldComparatorSource extends FieldComparatorSource {
+	@Override
+	public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed)
+			throws IOException {
+		return new SimilarValuesFieldComparator(fieldname, numHits);
+	}
+}

search-console/src/main/java/com/uas/search/console/util/DocumentToObjectUtils.java

@@ -83,8 +83,19 @@ public class DocumentToObjectUtils {
 		component.setId(Long.valueOf(document.get(SearchConstants.COMPONENT_ID_FIELD)));
 		component.setCode(document.get(SearchConstants.COMPONENT_CODE_FIELD));
 		component.setUuid(document.get(SearchConstants.COMPONENT_UUID_FIELD));
-		component.setKindid(Long.valueOf(document.get(SearchConstants.COMPONENT_KINDID_FIELD)));
-		component.setBrandid(Long.valueOf(document.get(SearchConstants.COMPONENT_BRANDID_FIELD)));
+
+		KindSimpleInfo kind = new KindSimpleInfo();
+		kind.setId(Long.valueOf(document.get(SearchConstants.COMPONENT_KINDID_FIELD)));
+		kind.setNameCn(document.get(SearchConstants.COMPONENT_KINDNAME_FIELD));
+		kind.setLevel(Short.valueOf(document.get(SearchConstants.COMPONENT_KINDLEVEL_FIELD)));
+		component.setKind(kind);
+
+		BrandSimpleInfo brand = new BrandSimpleInfo();
+		brand.setId(Long.valueOf(document.get(SearchConstants.COMPONENT_BRANDID_FIELD)));
+		brand.setNameCn(document.get(SearchConstants.COMPONENT_BRANDNAMECN_FIELD));
+		brand.setNameEn(document.get(SearchConstants.COMPONENT_BRANDNAMEEN_FIELD));
+		component.setBrand(brand);
+
 		if (!StringUtils.isEmpty(document.get(SearchConstants.COMPONENT_RESERVE_FIELD))) {
 			component.setReserve(Double.valueOf(document.get(SearchConstants.COMPONENT_RESERVE_FIELD)));
 		}

search-console/src/main/java/com/uas/search/console/util/MergeComponentData.java

@@ -1,18 +1,26 @@
 package com.uas.search.console.util;
 
 import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.PrintWriter;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 import org.springframework.util.StringUtils;
 
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.uas.search.console.model.BrandSimpleInfo;
 import com.uas.search.console.model.ComponentSimpleInfo;
+import com.uas.search.console.model.KindSimpleInfo;
 import com.uas.search.console.model.PropertyValueSimpleInfo;
 
 /**
@@ -27,7 +35,25 @@ public class MergeComponentData {
 	public static final int SINGLE_FILE_MAX_SIZE = 100000;
 
 	// 器件和属性值两个文本文件的路径
-	private static final String DATA_DIR = "C:\\Users\\sunyj-pc\\Desktop\\data\\prod";
+	private static final String DATA_DIR = "C:\\Users\\sunyj-pc\\Desktop\\temp\\data\\test";
+	
+	private static Map<Long, KindSimpleInfo> kinds;
+	private static Map<Long, BrandSimpleInfo> brands ;
+	
+	static{
+		try {
+			File kindFile=new File(DATA_DIR+"/kind.json");
+			String json = readAsString(kindFile);
+			kinds = parseKinds(json);
+			
+			File brandFile=new File(DATA_DIR+"/brand.json");
+			String json2 = readAsString(brandFile);
+			 brands = parseBrands(json2);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+	
 
 	/**
 	 * 将从本地文件读取的一行字符串数据解析为器件对象
@@ -36,29 +62,76 @@ public class MergeComponentData {
 	 *            格式为"CMP_ID" "CMP_BRID" "CMP_CODE" "CMP_KIID" "CMP_RESERVE"
 	 *            "CMP_SAMPLEQTY" "CMP_ORIGINALQTY" "CMP_UUID" "CMP_INASTOCKQTY"
 	 * @return
+	 * @throws IOException 
+	 * @throws FileNotFoundException 
 	 */
-	private static ComponentSimpleInfo parseComponent(String data) {
+	private static ComponentSimpleInfo parseComponent(String data) throws FileNotFoundException, IOException {
+		
+		
+		
 		ComponentSimpleInfo component = new ComponentSimpleInfo();
 		String[] strs = data.split("\t");
 		component.setId(Long.parseLong(strs[0]));
-		component.setBrandid(Long.parseLong(strs[1]));
+		long brandId = Long.parseLong(strs[1]);
+		component.setBrand(brands.get(brandId));
 		component.setCode(strs[2].substring(1, strs[2].length() - 1));
-		if (!StringUtils.isEmpty(strs[3])) {
-			component.setReserve(Double.valueOf(strs[3]));
-		}
+		Long kindId=Long.parseLong(strs[3]);
+		component.setKind(kinds.get(kindId));
 		if (!StringUtils.isEmpty(strs[4])) {
-			component.setSampleQty(Double.valueOf(strs[4]));
+			component.setReserve(Double.valueOf(strs[4]));
 		}
 		if (!StringUtils.isEmpty(strs[5])) {
-			component.setOriginalQty(Double.valueOf(strs[5]));
+			component.setSampleQty(Double.valueOf(strs[5]));
 		}
 		if (!StringUtils.isEmpty(strs[6])) {
-			component.setInactionStockQty(Double.valueOf(strs[6]));
+			component.setOriginalQty(Double.valueOf(strs[6]));
+		}
+		component.setUuid(strs[7].substring(1, strs[7].length() - 1));
+		if (strs.length>8 && !StringUtils.isEmpty(strs[8])) {
+			component.setInactionStockQty(Double.valueOf(strs[8]));
 		}
-		component.setKindid(Long.parseLong(strs[7]));
-		component.setUuid(strs[8].substring(1, strs[8].length() - 1));
 		return component;
 	}
+	
+	private static Map<Long, KindSimpleInfo> parseKinds(String json){
+		JSONObject jsonObject = JSONObject.parseObject(json);
+		JSONArray jsonArray = jsonObject.getJSONArray("items");
+		Map<Long, KindSimpleInfo> map=new HashMap<>();
+		for(Object object :jsonArray){
+			JSONObject obj=(JSONObject) object;
+			Long id = obj.getLong("ki_id");
+			String name = obj.getString("ki_name");
+			Short level = obj.getShort("ki_level");
+			Short isLeaf = obj.getShort("ki_isleaf");
+			KindSimpleInfo kind=new KindSimpleInfo();
+			kind.setId(id);
+			kind.setNameCn(name);
+			kind.setLevel(level);
+			kind.setIsLeaf(isLeaf);
+			map.put(id, kind);
+		}
+		return map;
+	}
+	
+	private static Map<Long, BrandSimpleInfo> parseBrands(String json){
+		JSONObject jsonObject = JSONObject.parseObject(json);
+		JSONArray jsonArray = jsonObject.getJSONArray("items");
+		Map<Long, BrandSimpleInfo> map=new HashMap<>();
+		for(Object object :jsonArray){
+			JSONObject obj=(JSONObject) object;
+			Long id = obj.getLong("br_id");
+			String uuid = obj.getString("br_uuid");
+			String nameCn = obj.getString("br_name_cn");
+			String nameEn = obj.getString("br_name_en");
+			BrandSimpleInfo brand=new BrandSimpleInfo();
+			brand.setId(id);
+			brand.setUuid(uuid);
+			brand.setNameCn(nameCn);
+			brand.setNameEn(nameEn);
+			map.put(id, brand);
+		}
+		return map;
+	}
 
 	/**
 	 * 将从本地文件读取的一行字符串数据解析为属性值对象
@@ -76,6 +149,57 @@ public class MergeComponentData {
 		propertyValue.setValue(strs[2].substring(1, strs[2].length() - 1));
 		return propertyValue;
 	}
+	
+	/**
+	 * 读取文件内容输出字符串
+	 * 
+	 * @param file
+	 *            文件
+	 * @return 读取的内容
+	 */
+	public static String readAsString( File file) throws FileNotFoundException, IOException {
+		if (!file.exists() || file.isDirectory()) {
+			return null;
+		}
+		FileInputStream stream = new FileInputStream(file);
+		return readAsString(stream);
+	}
+
+	/**
+	 * 读取输入流输出字符串
+	 * 
+	 * @param stream
+	 *            将要读的输入流
+	 * @return 读取的内容
+	 */
+	public static String readAsString( InputStream stream)
+			throws FileNotFoundException, IOException {
+		// 以UTF-8编码方式读取文件
+		String result = new String(readStream(stream), "UTF-8");
+		stream.close();
+		return result;
+	}
+
+	/**
+	 * Read the supplied InputStream and return as a byte array.
+	 * 
+	 * @param stream
+	 *            The stream to read.
+	 * @return byte array containing the Stream data.
+	 * @throws IOException
+	 *             Exception reading from the stream.
+	 */
+	public static byte[] readStream( InputStream stream) throws IOException {
+		ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+		byte[] byteBuf = new byte[1024];
+		int readCount = 0;
+
+		while ((readCount = stream.read(byteBuf)) != -1) {
+			bytesOut.write(byteBuf, 0, readCount);
+		}
+		stream.close();
+		return bytesOut.toByteArray();
+	}
 
 	public static void merge() {
 		BufferedReader componentReader = null;
@@ -169,9 +293,8 @@ public class MergeComponentData {
 		}
 	}
 
-	// public static void main(String[] args) {
-	// merge();
-	// // readData();
-	// }
+//	public static void main(String[] args) {
+//		merge();
+//	}
 
 }

search-console/src/main/java/com/uas/search/console/util/ObjectToDocumentUtils.java

@@ -3,12 +3,14 @@ package com.uas.search.console.util;
 import java.util.Set;
 
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DoubleField;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.util.BytesRef;
 import org.springframework.util.StringUtils;
 
 import com.alibaba.fastjson.JSONObject;
@@ -115,8 +117,8 @@ public class ObjectToDocumentUtils {
 	 */
 	public static Document toDocument(ComponentSimpleInfo component) {
 		if (component == null || component.getId() == null || StringUtils.isEmpty(component.getUuid())
-				|| StringUtils.isEmpty(component.getCode()) || component.getKindid() == null
-				|| component.getBrandid() == null) {
+				|| StringUtils.isEmpty(component.getCode()) || component.getKind() == null
+				|| component.getBrand() == null) {
 			return null;
 		}
 		Document document = new Document();
@@ -125,9 +127,18 @@ public class ObjectToDocumentUtils {
 		// 转小写，以避免分词，又不会因大小写影响搜索
 		String code = component.getCode().toLowerCase();
 		document.add(new StringField(SearchConstants.COMPONENT_CODE_FIELD, code, Store.YES));
-		document.add(new StringField(SearchConstants.COMPONENT_KINDID_FIELD, String.valueOf(component.getKindid()),
+		document.add(new BinaryDocValuesField(SearchConstants.COMPONENT_CODE_FIELD, new BytesRef(code)));
+		document.add(new StringField(SearchConstants.COMPONENT_KINDID_FIELD,
+				String.valueOf(component.getKind().getId()), Store.YES));
+		document.add(
+				new TextField(SearchConstants.COMPONENT_KINDNAME_FIELD, component.getKind().getNameCn(), Store.YES));
+		document.add(new StringField(SearchConstants.COMPONENT_KINDLEVEL_FIELD,
+				String.valueOf(component.getKind().getLevel()), Store.YES));
+		document.add(new StringField(SearchConstants.COMPONENT_BRANDID_FIELD,
+				String.valueOf(component.getBrand().getId()), Store.YES));
+		document.add(new TextField(SearchConstants.COMPONENT_BRANDNAMECN_FIELD, component.getBrand().getNameCn(),
 				Store.YES));
-		document.add(new StringField(SearchConstants.COMPONENT_BRANDID_FIELD, String.valueOf(component.getBrandid()),
+		document.add(new TextField(SearchConstants.COMPONENT_BRANDNAMEEN_FIELD, component.getBrand().getNameEn(),
 				Store.YES));
 		if (component.getReserve() != null) {
 			document.add(new DoubleField(SearchConstants.COMPONENT_RESERVE_FIELD, component.getReserve(), Store.YES));

search-console/src/main/java/com/uas/search/console/util/SearchConstants.java

@@ -92,13 +92,15 @@ public class SearchConstants {
 	public static final String COMPONENT_CODE_FIELD = "cmp_code";
 	public static final String COMPONENT_KINDID_FIELD = "cmp_kind_id";
 	public static final String COMPONENT_KINDNAME_FIELD = "cmp_kind_name_cn";
+	public static final String COMPONENT_KINDLEVEL_FIELD = "cmp_kind_level";
 	public static final String COMPONENT_BRANDID_FIELD = "cmp_brand_id";
+	public static final String COMPONENT_BRANDNAMECN_FIELD = "cmp_brand_name_cn";
+	public static final String COMPONENT_BRANDNAMEEN_FIELD = "cmp_brand_name_en";
 	public static final String COMPONENT_RESERVE_FIELD = "cmp_reserve";
 	public static final String COMPONENT_SAMPLE_QTY_FIELD = "cmp_sample_qty";
 	public static final String COMPONENT_ORIGINAL_QTY_FIELD = "cmp_original_qty";
 	public static final String COMPONENT_INACTION_STOCK_QTY_FIELD = "cmp_inaction_stock_qty";
 	public static final String COMPONENT_BRANDUUID_FIELD = "cmp_brand_uuid";
-	public static final String COMPONENT_BRANDNAME_FIELD = "cmp_brand_name_cn";
 	/**
 	 * 创建器件属性的索引时，在属性id的前面添加该前缀作为Field的键
 	 */