package com.uas.search.service.impl; import java.io.IOException; import java.net.URLDecoder; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import com.uas.search.annotation.NotEmpty; import org.apache.lucene.document.Document; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortField.Type; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import com.alibaba.fastjson.JSONObject; import com.uas.search.constant.SearchConstants; import com.uas.search.constant.model.CollectField; import com.uas.search.constant.model.PageParams; import com.uas.search.constant.model.PageParams.FilterField; import com.uas.search.constant.model.SPage; import com.uas.search.exception.SearchException; import com.uas.search.grouping.DistinctGroupCollector; import com.uas.search.grouping.GoodsGroupCollector; import com.uas.search.model.Brand; import com.uas.search.model.Component; import com.uas.search.model.Goods; import com.uas.search.model.Kind; import com.uas.search.service.SearchService; import com.uas.search.sort.SimilarValuesFieldComparatorSource; import com.uas.search.util.DocumentToObjectUtils; import com.uas.search.util.SearchUtils; /** * 搜索索引 * * @author sunyj * @since 2016年8月5日 下午2:21:26 */ @Service public class SearchServiceImpl implements SearchService { /** * 获取联想词时返回的最大数目 */ private static final int SIMILAR_NUM = 20; private static Logger logger = LoggerFactory.getLogger(SearchServiceImpl.class); @Override public SPage getKindIds(String keyword, Integer page, Integer size) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("搜索关键词无效:" + keyword); } List ids = new ArrayList(); BooleanQuery booleanQuery = SearchUtils.getBooleanQuery(SearchConstants.KIND_NAMECN_FIELD, keyword); logger.info(booleanQuery.toString()); SPage documents = SearchUtils.getDocuments(SearchConstants.KIND_TABLE_NAME, booleanQuery, page, size); SPage sPage = new SPage(documents.getTotalPage(), documents.getTotalElement(), documents.getPage(), documents.getSize(), documents.isFirst(), documents.isLast()); for (Document document : documents.getContent()) { ids.add(Long.parseLong(document.get(SearchConstants.KIND_ID_FIELD))); } sPage.setContent(ids); return sPage; } @Override public SPage> getKinds(String keyword, Integer page, Integer size) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("搜索关键词无效:" + keyword); } List> kinds = new ArrayList>(); BooleanQuery booleanQuery = SearchUtils.getBooleanQuery(SearchConstants.KIND_NAMECN_FIELD, keyword); logger.info(booleanQuery.toString()); SPage documents = SearchUtils.getDocuments(SearchConstants.KIND_TABLE_NAME, booleanQuery, page, size); SPage> sPage = new SPage>(documents.getTotalPage(), documents.getTotalElement(), documents.getPage(), documents.getSize(), documents.isFirst(), documents.isLast()); for (Document document : documents.getContent()) { Map kind = new HashMap(); kind.put("id", Long.parseLong(document.get(SearchConstants.KIND_ID_FIELD))); kind.put("nameCn", document.get(SearchConstants.KIND_NAMECN_FIELD)); kinds.add(kind); } sPage.setContent(kinds); return sPage; } @Override public SPage getBrandIds(String keyword, Integer page, Integer size) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("搜索关键词无效:" + keyword); } List ids = new ArrayList(); BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.BRAND_NAMECN_FIELD, keyword), BooleanClause.Occur.SHOULD); booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.BRAND_NAMEEN_FIELD, keyword), BooleanClause.Occur.SHOULD); logger.info(booleanQuery.toString()); SPage documents = SearchUtils.getDocuments(SearchConstants.BRAND_TABLE_NAME, booleanQuery, page, size); SPage sPage = new SPage(documents.getTotalPage(), documents.getTotalElement(), documents.getPage(), documents.getSize(), documents.isFirst(), documents.isLast()); for (Document document : documents.getContent()) { ids.add(Long.parseLong(document.get(SearchConstants.BRAND_ID_FIELD))); } sPage.setContent(ids); return sPage; } @Override public SPage> getBrands(String keyword, Integer page, Integer size) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("搜索关键词无效:" + keyword); } List> brands = new ArrayList>(); BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.BRAND_NAMECN_FIELD, keyword), BooleanClause.Occur.SHOULD); booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.BRAND_NAMEEN_FIELD, keyword), BooleanClause.Occur.SHOULD); logger.info(booleanQuery.toString()); SPage documents = SearchUtils.getDocuments(SearchConstants.BRAND_TABLE_NAME, booleanQuery, page, size); SPage> sPage = new SPage>(documents.getTotalPage(), documents.getTotalElement(), documents.getPage(), documents.getSize(), documents.isFirst(), documents.isLast()); for (Document document : documents.getContent()) { Map brand = new HashMap(); brand.put("id", Long.parseLong(document.get(SearchConstants.BRAND_ID_FIELD))); brand.put("uuid", document.get(SearchConstants.BRAND_UUID_FIELD)); brand.put("nameCn", document.get(SearchConstants.BRAND_NAMECN_FIELD)); brand.put("nameEn", document.get(SearchConstants.BRAND_NAMEEN_FIELD)); brands.add(brand); } sPage.setContent(brands); return sPage; } @Override public Map getComponentIds(String keyword, PageParams pageParams) { Map searchComponentIds = getComponentIds(keyword, pageParams, null, null); return searchComponentIds; // TODO 对品牌、类目甚至拼音混合搜索(待完善) // int total = (int) searchComponentIds.get("total"); // if (total != 0) { // return searchComponentIds; // } // List kindIds = getKindIds(keyword, Occur.SHOULD); // List brandIds = getBrandIds(keyword, Occur.SHOULD); // return getComponentIds(null, pageParams, kindIds, brandIds); } /** * 根据关键词搜索产品 * * @param keyword * @param pageParams * @param kindIds * @param brandIds * @return * @throws SearchException */ private Map getComponentIds(String keyword, PageParams pageParams, List kindIds, List brandIds) throws SearchException { // 因为器件、属性值的数据量远比类目、品牌大得多,而且器件搜索可能还需进行分页, // 所以涉及器件、属性值的搜索,大都不能像类目和品牌一样直接利用SearchUtils.getDocuments方法 IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.COMPONENT_TABLE_NAME); if (pageParams == null) { pageParams = new PageParams(); } if (pageParams.getPage() <= 0) pageParams.setPage(1); if (pageParams.getSize() <= 0) pageParams.setSize(20); Map map = new HashMap(); List ids = new ArrayList(); try { BooleanQuery booleanQuery = new BooleanQuery(); if (!SearchUtils.isKeywordInvalid(keyword)) { booleanQuery.add(setBoost(keyword), BooleanClause.Occur.MUST); } Map filters = pageParams.getFilters(); if (!CollectionUtils.isEmpty(filters)) { // 筛选类目 if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_KINDID))) { String kindId = filters.get(FilterField.COMPONENT_KINDID).toString(); TermQuery kindQuery = new TermQuery(new Term(SearchConstants.COMPONENT_KINDID_FIELD, kindId)); booleanQuery.add(kindQuery, BooleanClause.Occur.MUST); } // 筛选品牌 if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_BRANDID))) { String brandId = filters.get(FilterField.COMPONENT_BRANDID).toString(); TermQuery brandQuery = new TermQuery(new Term(SearchConstants.COMPONENT_BRANDID_FIELD, brandId)); booleanQuery.add(brandQuery, BooleanClause.Occur.MUST); } // 库存不为0 if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_RESERVE))) { Boolean isReserveNotEmpty = (Boolean) filters.get(FilterField.COMPONENT_HAS_RESERVE); if (isReserveNotEmpty) { booleanQuery.add(NumericRangeQuery.newDoubleRange(SearchConstants.COMPONENT_RESERVE_FIELD, 0.0, Double.MAX_VALUE, false, true), BooleanClause.Occur.MUST); } } // 现货、呆滞库存、样品数量不为0,取或的关系 if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_SAMPLE)) || !StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_ORIGINAL)) || !StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_INACTION_STOCK))) { BooleanQuery booleanQuery2 = new BooleanQuery(); if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_SAMPLE))) { booleanQuery2.add(NumericRangeQuery.newDoubleRange(SearchConstants.COMPONENT_SAMPLE_QTY_FIELD, 0.0, Double.MAX_VALUE, false, true), BooleanClause.Occur.SHOULD); } if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_ORIGINAL))) { booleanQuery2.add(NumericRangeQuery.newDoubleRange(SearchConstants.COMPONENT_ORIGINAL_QTY_FIELD, 0.0, Double.MAX_VALUE, false, true), BooleanClause.Occur.SHOULD); } if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_HAS_INACTION_STOCK))) { booleanQuery2.add( NumericRangeQuery.newDoubleRange(SearchConstants.COMPONENT_INACTION_STOCK_QTY_FIELD, 0.0, Double.MAX_VALUE, false, true), BooleanClause.Occur.SHOULD); } booleanQuery.add(booleanQuery2, Occur.MUST); } // 属性过滤 if (!StringUtils.isEmpty(filters.get(FilterField.COMPONENT_PROPERTIES))) { JSONObject proJSON = JSONObject .parseObject(String.valueOf(filters.get(FilterField.COMPONENT_PROPERTIES))); for (String key : proJSON.keySet()) { String value = String.valueOf(proJSON.get(key)); if (!StringUtils.isEmpty(value)) { if (!key.startsWith(SearchConstants.COMPONENT_PROPERTY_PREFIX)) { key = SearchConstants.COMPONENT_PROPERTY_PREFIX + key; } TermQuery propertyQuery = new TermQuery(new Term(key, value)); booleanQuery.add(propertyQuery, BooleanClause.Occur.MUST); } } } } if (!CollectionUtils.isEmpty(kindIds)) { BooleanQuery booleanQuery2 = new BooleanQuery(); for (Long id : kindIds) { booleanQuery2.add(new TermQuery(new Term(SearchConstants.COMPONENT_KINDID_FIELD, id.toString())), Occur.SHOULD); } booleanQuery.add(booleanQuery2, Occur.MUST); } if (!CollectionUtils.isEmpty(brandIds)) { BooleanQuery booleanQuery2 = new BooleanQuery(); for (Long id : brandIds) { booleanQuery2.add(new TermQuery(new Term(SearchConstants.COMPONENT_BRANDID_FIELD, id.toString())), Occur.SHOULD); } booleanQuery.add(booleanQuery2, Occur.MUST); } logger.info(booleanQuery.toString()); Sort sort = new Sort(SortField.FIELD_SCORE); TopDocs hits; if (pageParams.getPage() > 1) {// 不是第一页 TopDocs previousHits = indexSearcher.search(booleanQuery, (pageParams.getPage() - 1) * pageParams.getSize(), sort, true, false); ScoreDoc[] previousScoreDocs = previousHits.scoreDocs; ScoreDoc after = previousScoreDocs[previousScoreDocs.length - 1]; hits = indexSearcher.searchAfter(after, booleanQuery, pageParams.getSize(), sort, true, false); } else { hits = indexSearcher.search(booleanQuery, pageParams.getSize(), sort, true, false); } ScoreDoc[] scoreDocs = hits.scoreDocs; for (ScoreDoc scoreDoc : scoreDocs) { // 数据量太大,需要指定将获取的数据(以免载入不必要的数据,降低速度) Set fieldsToLoad = new HashSet<>(); fieldsToLoad.add(SearchConstants.COMPONENT_ID_FIELD); Document document = indexSearcher.doc(scoreDoc.doc, fieldsToLoad); String componentId = document.get(SearchConstants.COMPONENT_ID_FIELD); ids.add(Long.parseLong(componentId)); // System.out.println(componentId + "\t" + scoreDoc.score); } map.put("componentIds", ids); map.put("page", pageParams.getPage()); map.put("size", pageParams.getSize()); map.put("total", hits.totalHits); } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return map; } /** * 同时搜索器件、类目、品牌,并设置boost * * @param keyword * @return */ private Query setBoost(String keyword) { BooleanQuery booleanQuery = new BooleanQuery(); PrefixQuery prefixQuery = new PrefixQuery( new Term(SearchConstants.COMPONENT_CODE_FIELD, keyword.toLowerCase())); prefixQuery.setBoost(100); booleanQuery.add(prefixQuery, BooleanClause.Occur.SHOULD); BooleanQuery brandNameCnQuery = SearchUtils.getBooleanQuery(SearchConstants.COMPONENT_BRANDNAMECN_FIELD, keyword); brandNameCnQuery.setBoost(10); booleanQuery.add(brandNameCnQuery, BooleanClause.Occur.SHOULD); BooleanQuery brandNameEnQuery = SearchUtils.getBooleanQuery(SearchConstants.COMPONENT_BRANDNAMEEN_FIELD, keyword); brandNameEnQuery.setBoost(10); booleanQuery.add(brandNameEnQuery, BooleanClause.Occur.SHOULD); BooleanQuery kindNameQuery = SearchUtils.getBooleanQuery(SearchConstants.COMPONENT_KINDNAME_FIELD, keyword); kindNameQuery.setBoost(1); booleanQuery.add(kindNameQuery, BooleanClause.Occur.SHOULD); return booleanQuery; } @Override public Set getKindIdsBySearchComponent(String keyword, String brandId) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("搜索关键词无效:" + keyword); } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.COMPONENT_TABLE_NAME); Set kindIds = new HashSet(); try { BooleanQuery booleanQuery = new BooleanQuery(); keyword = URLDecoder.decode(keyword, "UTF-8"); booleanQuery.add(setBoost(keyword), BooleanClause.Occur.MUST); // 筛选品牌 if (!StringUtils.isEmpty(brandId)) { TermQuery brandQuery = new TermQuery(new Term(SearchConstants.COMPONENT_BRANDID_FIELD, brandId)); booleanQuery.add(brandQuery, BooleanClause.Occur.MUST); } logger.info(booleanQuery.toString()); DistinctGroupCollector collector = new DistinctGroupCollector(SearchConstants.COMPONENT_KINDID_FIELD); indexSearcher.search(booleanQuery, collector); kindIds = collector.getValues(); } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return kindIds; } @Override public List> getKindsBySearchComponent(String keyword, String brandId) { Set kindIds = getKindIdsBySearchComponent(keyword, brandId); List> kinds = new ArrayList>(); if (CollectionUtils.isEmpty(kindIds)) { return kinds; } BooleanQuery booleanQuery = new BooleanQuery(); for (Long kindId : kindIds) { booleanQuery.add(new TermQuery(new Term(SearchConstants.KIND_ID_FIELD, String.valueOf(kindId))), BooleanClause.Occur.SHOULD); } List documents = SearchUtils.getDocuments(SearchConstants.KIND_TABLE_NAME, booleanQuery).getContent(); for (Document document : documents) { Map kind = new HashMap(); kind.put("id", Long.parseLong(document.get(SearchConstants.KIND_ID_FIELD))); kind.put("nameCn", document.get(SearchConstants.KIND_NAMECN_FIELD)); kinds.add(kind); } return kinds; } @Override public Set getBrandIdsBySearchComponent(String keyword, String kindId) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("搜索关键词无效:" + keyword); } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.COMPONENT_TABLE_NAME); Set brandIds = new HashSet(); try { BooleanQuery booleanQuery = new BooleanQuery(); keyword = URLDecoder.decode(keyword, "UTF-8"); booleanQuery.add(setBoost(keyword), BooleanClause.Occur.MUST); // 筛选类目 if (!StringUtils.isEmpty(kindId)) { TermQuery kindQuery = new TermQuery(new Term(SearchConstants.COMPONENT_KINDID_FIELD, kindId)); booleanQuery.add(kindQuery, BooleanClause.Occur.MUST); } logger.info(booleanQuery.toString()); DistinctGroupCollector collector = new DistinctGroupCollector(SearchConstants.COMPONENT_BRANDID_FIELD); indexSearcher.search(booleanQuery, collector); brandIds = collector.getValues(); } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return brandIds; } @Override public List> getBrandsBySearchComponent(String keyword, String kindId) { Set brandIds = getBrandIdsBySearchComponent(keyword, kindId); List> brands = new ArrayList>(); if (CollectionUtils.isEmpty(brandIds)) { return brands; } BooleanQuery booleanQuery = new BooleanQuery(); for (Long brandId : brandIds) { booleanQuery.add(new TermQuery(new Term(SearchConstants.BRAND_ID_FIELD, String.valueOf(brandId))), BooleanClause.Occur.SHOULD); } List documents = SearchUtils.getDocuments(SearchConstants.BRAND_TABLE_NAME, booleanQuery) .getContent(); for (Document document : documents) { Map brand = new HashMap(); brand.put("id", Long.parseLong(document.get(SearchConstants.BRAND_ID_FIELD))); brand.put("uuid", document.get(SearchConstants.BRAND_UUID_FIELD)); brand.put("nameCn", document.get(SearchConstants.BRAND_NAMECN_FIELD)); brands.add(brand); } return brands; } @Override public List getSimilarKeywords(String keyword) { List result = new ArrayList<>(); // 相似的器件原厂型号数量足够,直接返回 List componentCodes = getSimilarComponentCodes(keyword); result.addAll(componentCodes); removeDuplicate(result); if (result.size() == SIMILAR_NUM) { return result; } // 获取相似类目 List kindNames = getSimilarKindNames(keyword); if (!CollectionUtils.isEmpty(kindNames)) { result.addAll(kindNames); removeDuplicate(result); // 如果总的数量超出SIMILAR_NUM,去除多余的元素 if (result.size() > SIMILAR_NUM) { removeElements(result, SIMILAR_NUM); return result; } } // 获取相似品牌 List brandNames = getSimilarBrandNames(keyword); if (!CollectionUtils.isEmpty(brandNames)) { result.addAll(brandNames); removeDuplicate(result); if (result.size() > SIMILAR_NUM) { removeElements(result, SIMILAR_NUM); return result; } } return result; } @Override public List> getSimilarComponents(String componentCode) { if (SearchUtils.isKeywordInvalid(componentCode)) { throw new SearchException("输入无效:" + componentCode); } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.COMPONENT_TABLE_NAME); List> components = new ArrayList<>(); try { PrefixQuery prefixQuery = new PrefixQuery( new Term(SearchConstants.COMPONENT_CODE_FIELD, componentCode.toLowerCase())); logger.info(prefixQuery.toString()); TopDocs hits = indexSearcher.search(prefixQuery, SIMILAR_NUM); ScoreDoc[] scoreDocs = hits.scoreDocs; for (ScoreDoc scoreDoc : scoreDocs) { Set fieldsToLoad = new HashSet<>(); fieldsToLoad.add(SearchConstants.COMPONENT_ID_FIELD); fieldsToLoad.add(SearchConstants.COMPONENT_UUID_FIELD); fieldsToLoad.add(SearchConstants.COMPONENT_CODE_FIELD); Document document = indexSearcher.doc(scoreDoc.doc, fieldsToLoad); Map map = new HashMap<>(); map.put("id", Long.parseLong(document.get(SearchConstants.COMPONENT_ID_FIELD))); map.put("uuid", document.get(SearchConstants.COMPONENT_UUID_FIELD)); map.put("code", document.get(SearchConstants.COMPONENT_CODE_FIELD)); components.add(map); } } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return components; } @Override public List> getSimilarBrands(String brandName) { if (SearchUtils.isKeywordInvalid(brandName)) { throw new SearchException("输入无效:" + brandName); } List> brands = new ArrayList>(); // 品牌名称带有空格,并且中英文名并无一定顺序,因此对nameCn、nameEn均要搜索 BooleanQuery booleanQuery = new BooleanQuery(); String[] keywords = brandName.split(" "); for (String keyword : keywords) { // 搜索nameCn booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.BRAND_NAMECN_FIELD, keyword), BooleanClause.Occur.SHOULD); // 搜索nameEn booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.BRAND_NAMEEN_FIELD, keyword), BooleanClause.Occur.SHOULD); } logger.info(booleanQuery.toString()); List documents = SearchUtils.getDocuments(SearchConstants.BRAND_TABLE_NAME, booleanQuery) .getContent(); for (Document document : documents) { Map brand = new HashMap<>(); brand.put("id", Long.parseLong(document.get(SearchConstants.BRAND_ID_FIELD))); brand.put("uuid", document.get(SearchConstants.BRAND_UUID_FIELD)); brand.put("nameCn", document.get(SearchConstants.BRAND_NAMECN_FIELD)); brand.put("nameEn", document.get(SearchConstants.BRAND_NAMEEN_FIELD)); brands.add(brand); } return brands; } @Override public List> getSimilarKinds(String kindName) { return getSimilarKinds(kindName, null, null); } @Override public List> getSimilarLeafKinds(String kindName) { return getSimilarKinds(kindName, (short) 1, null); } @Override public List> getSimilarKindsByLevel(String kindName, Short level) { return getSimilarKinds(kindName, null, level); } /** * 根据输入的类目名获取联想词 * * @param kindName * 类目名 * @param isLeaf * 是否只获取末级类目 * @param level * 指定的类目级别 * @return */ private List> getSimilarKinds(String kindName, Short isLeaf, Short level) { if (SearchUtils.isKeywordInvalid(kindName)) { throw new SearchException("输入无效:" + kindName); } List> kinds = new ArrayList<>(); BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.add(SearchUtils.getBooleanQuery(SearchConstants.KIND_NAMECN_FIELD, kindName), BooleanClause.Occur.MUST); if (isLeaf != null && isLeaf == 1) { booleanQuery.add(new TermQuery(new Term(SearchConstants.KIND_ISLEAF_FIELD, String.valueOf(isLeaf))), BooleanClause.Occur.MUST); } else { if (level != null && level > 0) { booleanQuery.add(new TermQuery(new Term(SearchConstants.KIND_LEVEL_FIELD, String.valueOf(level))), BooleanClause.Occur.MUST); } } logger.info(booleanQuery.toString()); List documents = SearchUtils.getDocuments(SearchConstants.KIND_TABLE_NAME, booleanQuery).getContent(); for (Document document : documents) { Map map = new HashMap<>(); map.put("id", Long.parseLong(document.get(SearchConstants.KIND_ID_FIELD))); map.put("nameCn", document.get(SearchConstants.KIND_NAMECN_FIELD)); map.put("isLeaf", Short.parseShort(document.get(SearchConstants.KIND_ISLEAF_FIELD))); map.put("level", Short.parseShort(document.get(SearchConstants.KIND_LEVEL_FIELD))); kinds.add(map); } return kinds; } @Override public List> getSimilarPropertyValues(Long kindId, Long propertyId, String keyword, Long topNum) { if (kindId == null || propertyId == null) { throw new SearchException("类目id和属性id不能为空"); } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.COMPONENT_TABLE_NAME); String propertyIdString = String.valueOf(propertyId); if (!propertyIdString.startsWith(SearchConstants.COMPONENT_PROPERTY_PREFIX)) { propertyIdString = SearchConstants.COMPONENT_PROPERTY_PREFIX + propertyIdString; } propertyIdString = propertyIdString + SearchConstants.COMPONENT_PROPERTY_TOKENIZED_SUFFIX; if (keyword == null) { keyword = ""; } if (topNum == null || topNum < 1) { topNum = (long) SIMILAR_NUM; } List propertyValues = new ArrayList<>(); try { BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.add(new TermQuery(new Term(SearchConstants.COMPONENT_KINDID_FIELD, String.valueOf(kindId))), BooleanClause.Occur.MUST); booleanQuery.add(new PrefixQuery(new Term(propertyIdString, keyword.toLowerCase())), BooleanClause.Occur.MUST); logger.info(booleanQuery.toString()); // 如果只搜索topNum个结果,去除重复的属性值后,数目很可能是不够的 TopDocs topDocs = indexSearcher.search(booleanQuery, SearchConstants.TOP_NUM); ScoreDoc[] scoreDocs = topDocs.scoreDocs; for (ScoreDoc scoreDoc : scoreDocs) { Set fieldsToLoad = new HashSet<>(); fieldsToLoad.add(propertyIdString); Document document = indexSearcher.doc(scoreDoc.doc, fieldsToLoad); String propertyValue = document.get(propertyIdString); if (!StringUtils.isEmpty(propertyValue) && !propertyValues.contains(propertyValue)) { propertyValues.add(propertyValue); } if (propertyValues.size() >= topNum) { break; } } } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } List> result = new ArrayList<>(); for (String propertyValue : propertyValues) { Map map = new HashMap<>(); map.put("propertyValue", propertyValue); result.add(map); } return result; } /** * 根据输入获取相似的器件原厂型号 * * @param componentCode * @return */ private List getSimilarComponentCodes(String componentCode) { return getSimilarValues(SearchConstants.COMPONENT_TABLE_NAME, SearchConstants.COMPONENT_CODE_FIELD, componentCode.toLowerCase(), QueryType.PREFIX_QUERY); } /** * 根据输入获取相似的品牌名称 * * @param brandName * @return */ private List getSimilarBrandNames(String brandName) { // 获取相似的中文品牌 List nameCns = getSimilarValues(SearchConstants.BRAND_TABLE_NAME, SearchConstants.BRAND_NAMECN_FIELD, brandName, QueryType.BOOLEAN_QUERY); // 相似的中文品牌数量足够,直接返回 if (nameCns != null && nameCns.size() == SIMILAR_NUM) { return nameCns; } List names = nameCns; // 获取相似的英文品牌 List nameEns = getSimilarValues(SearchConstants.BRAND_TABLE_NAME, SearchConstants.BRAND_NAMEEN_FIELD, brandName, QueryType.BOOLEAN_QUERY); names.addAll(nameEns); return names; } /** * 根据输入获取相似的类目名称 * * @param kindName * @return */ private List getSimilarKindNames(String kindName) { return getSimilarValues(SearchConstants.KIND_TABLE_NAME, SearchConstants.KIND_NAMECN_FIELD, kindName, QueryType.BOOLEAN_QUERY); } private enum QueryType { BOOLEAN_QUERY, PREFIX_QUERY } /** * 根据输入值获取该域相似的值 * * @param tableName * @param field * @param keyword * @return */ private List getSimilarValues(String tableName, String field, String keyword, QueryType queryType) { if (SearchUtils.isKeywordInvalid(keyword)) { throw new SearchException("输入无效:" + keyword); } if (queryType == null) { queryType = QueryType.BOOLEAN_QUERY; } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(tableName); List result = new ArrayList<>(); try { Query query = null; if (queryType == QueryType.BOOLEAN_QUERY) { query = SearchUtils.getBooleanQuery(field, keyword); } else if (queryType == QueryType.PREFIX_QUERY) { query = new PrefixQuery(new Term(field, keyword)); } logger.info(query.toString()); Sort sort = new Sort(new SortField(field, new SimilarValuesFieldComparatorSource())); TopDocs hits = indexSearcher.search(query, SIMILAR_NUM, sort, true, false); ScoreDoc[] scoreDocs = hits.scoreDocs; for (ScoreDoc scoreDoc : scoreDocs) { Set fieldsToLoad = new HashSet<>(); fieldsToLoad.add(field); Document document = indexSearcher.doc(scoreDoc.doc, fieldsToLoad); result.add(document.get(field)); // System.out.println(document.get(field) + "\t" + // scoreDoc.score); } } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return result; } /** * 移除集合中重复的元素 * * @param list * @return */ private void removeDuplicate(List list) { if (list == null) { return; } List result = new ArrayList<>(); for (String str : list) { if (!result.contains(str)) { result.add(str); } } list.removeAll(list); list.addAll(result); } /** * 删除collection内startIndex(含)后的元素 * * @param collection * @param startIndex */ private void removeElements(Collection collection, int startIndex) { if (CollectionUtils.isEmpty(collection)) { return; } int listsSize = collection.size(); for (int i = listsSize - 1; i >= startIndex; i--) { collection.remove(i); } } @Override public Map getGoodsIds(String keyword, PageParams pageParams) throws SearchException { List keywordFields = new ArrayList<>(); keywordFields.add(SearchConstants.GOODS_BR_NAME_CN_UNTOKENIZED_FIELD); keywordFields.add(SearchConstants.GOODS_BR_NAME_EN_UNTOKENIZED_FIELD); Map goodsIds = getGoodsIds(keyword, keywordFields, false, pageParams); if (CollectionUtils.isEmpty(goodsIds) || goodsIds.get("componentIds") == null || JSONObject.parseArray(goodsIds.get("componentIds").toString()).isEmpty()) { goodsIds = getGoodsIds(keyword, null, true, pageParams); } return goodsIds; } /** * @param keyword * @param keywordFields * 要查询的字段 * @param tokenized * 是否分词 * @param pageParams * @return * @throws SearchException */ private Map getGoodsIds(String keyword, List keywordFields, Boolean tokenized, PageParams pageParams) throws SearchException { // 因为器件、属性值的数据量远比类目、品牌大得多,而且器件搜索可能还需进行分页, // 所以涉及器件、属性值的搜索,大都不能像类目和品牌一样直接利用SearchUtils.getDocuments方法 IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.GOODS_TABLE_NAME); if (pageParams == null) { pageParams = new PageParams(); } if (pageParams.getPage() <= 0) pageParams.setPage(1); if (pageParams.getSize() <= 0) pageParams.setSize(20); Map map = new HashMap(); List cmpIds = new ArrayList<>(); List goIds = new ArrayList<>(); try { BooleanQuery booleanQuery = queryGoods(keyword, keywordFields, tokenized); Map filters = pageParams.getFilters(); if (!CollectionUtils.isEmpty(filters)) { // 筛选类目 if (!StringUtils.isEmpty(filters.get(FilterField.GOODS_KINDID))) { Object kindIds = filters.get(FilterField.GOODS_KINDID); filter(kindIds, SearchConstants.GOODS_KI_ID_FIELD, booleanQuery); } // 筛选品牌 if (!StringUtils.isEmpty(filters.get(FilterField.GOODS_BRANDID))) { Object brandIds = filters.get(FilterField.GOODS_BRANDID); filter(brandIds, SearchConstants.GOODS_BR_ID_FIELD, booleanQuery); } // 筛选货源 if (!StringUtils.isEmpty(filters.get(FilterField.GOODS_STORE_TYPE))) { Object storeTypes = filters.get(FilterField.GOODS_STORE_TYPE); filter(storeTypes, SearchConstants.GOODS_ST_TYPE_FIELD, booleanQuery); } // 筛选货币 if (!StringUtils.isEmpty(filters.get(FilterField.GOODS_CRNAME))) { Object crNames = filters.get(FilterField.GOODS_CRNAME); filter(crNames, SearchConstants.GOODS_CRNAME_FIELD, booleanQuery); } // 价格筛选 Object minPriceRmb = filters.get(FilterField.GOODS_MINPRICERMB); Object maxPriceRmb = filters.get(FilterField.GOODS_MAXPRICERMB); Object minPriceUsd = filters.get(FilterField.GOODS_MINPRICEUSD); Object maxPriceUsd = filters.get(FilterField.GOODS_MAXPRICEUSD); // 筛选人民币价格 if (!StringUtils.isEmpty(minPriceRmb) || !StringUtils.isEmpty(maxPriceRmb)) { Double minPrice = null; Double maxPrice = null; if (!StringUtils.isEmpty(minPriceRmb)) { minPrice = Double.valueOf(minPriceRmb.toString()); } if (!StringUtils.isEmpty(maxPriceRmb)) { maxPrice = Double.valueOf(maxPriceRmb.toString()); } booleanQuery.add(NumericRangeQuery.newDoubleRange(SearchConstants.GOODS_GO_MINPRICERMB_FIELD, minPrice, maxPrice, true, true), BooleanClause.Occur.FILTER); } // 筛选美元价格 if (!StringUtils.isEmpty(minPriceUsd) || !StringUtils.isEmpty(maxPriceUsd)) { Double minPrice = null; Double maxPrice = null; if (!StringUtils.isEmpty(minPriceUsd)) { minPrice = Double.valueOf(minPriceUsd.toString()); } if (!StringUtils.isEmpty(maxPriceUsd)) { maxPrice = Double.valueOf(maxPriceUsd.toString()); } booleanQuery.add(NumericRangeQuery.newDoubleRange(SearchConstants.GOODS_GO_MINPRICEUSD_FIELD, minPrice, maxPrice, true, true), BooleanClause.Occur.FILTER); } } logger.info(booleanQuery.toString()); // 排序 Sort sort; List sorts = pageParams.getSort(); if (sorts != null && !CollectionUtils.isEmpty(sorts)) { SortField[] sortFields = new SortField[sorts.size()]; int i = 0; for (com.uas.search.constant.model.Sort s : sorts) { if (s.getField() == null) { throw new SearchException("排序字段不可为空:" + s); } switch (s.getField()) { // 价格 case GO_RESERVE: sortFields[i++] = new SortField(SearchConstants.GOODS_GO_RESERVE_FIELD, Type.DOUBLE, s.isReverse()); break; // 人民币价格 case GO_MINPRICERMB: sortFields[i++] = new SortField(SearchConstants.GOODS_GO_MINPRICERMB_FIELD, Type.DOUBLE, s.isReverse()); break; // 美元价格 case GO_MINPRICEUSD: sortFields[i++] = new SortField(SearchConstants.GOODS_GO_MINPRICEUSD_FIELD, Type.DOUBLE, s.isReverse()); break; // 打分 case GO_SEARCH: sortFields[i++] = (SortField.FIELD_SCORE); break; default: throw new SearchException("不支持该排序方式:" + s.getField()); } } sort = new Sort(sortFields); } else { sort = new Sort(SortField.FIELD_SCORE); } TopDocs hits; if (pageParams.getPage() > 1) {// 不是第一页 TopDocs previousHits = indexSearcher.search(booleanQuery, (pageParams.getPage() - 1) * pageParams.getSize(), sort, true, false); int totalHits = previousHits.totalHits; if ((pageParams.getPage() - 1) * pageParams.getSize() >= totalHits) { return map; } ScoreDoc[] previousScoreDocs = previousHits.scoreDocs; ScoreDoc after = previousScoreDocs[previousScoreDocs.length - 1]; hits = indexSearcher.searchAfter(after, booleanQuery, pageParams.getSize(), sort, true, false); } else { hits = indexSearcher.search(booleanQuery, pageParams.getSize(), sort, true, false); } ScoreDoc[] scoreDocs = hits.scoreDocs; for (ScoreDoc scoreDoc : scoreDocs) { // 数据量太大,需要指定将获取的数据(以免载入不必要的数据,降低速度) Set fieldsToLoad = new HashSet<>(); fieldsToLoad.add(SearchConstants.GOODS_CMP_ID_FIELD); fieldsToLoad.add(SearchConstants.GOODS_GO_ID_FIELD); Document document = indexSearcher.doc(scoreDoc.doc, fieldsToLoad); String cmpId = document.get(SearchConstants.GOODS_CMP_ID_FIELD); cmpIds.add(StringUtils.isEmpty(cmpId) ? null : Long.valueOf(cmpId)); String goId = document.get(SearchConstants.GOODS_GO_ID_FIELD); goIds.add(StringUtils.isEmpty(goId) ? null : Long.valueOf(goId)); // System.out.println(cmpId + "\t" + goId + "\t" + // scoreDoc.score); // System.out.println(indexSearcher.explain(booleanQuery, // scoreDoc.doc).toString()); } map.put("componentIds", cmpIds); map.put("goodsIds", goIds); map.put("page", pageParams.getPage()); map.put("size", pageParams.getSize()); map.put("total", hits.totalHits); } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return map; } @Override public List> collectBySearchGoods(String keyword, CollectField collectedField, Map filters) { List keywordFields = new ArrayList<>(); keywordFields.add(SearchConstants.GOODS_BR_NAME_CN_UNTOKENIZED_FIELD); keywordFields.add(SearchConstants.GOODS_BR_NAME_EN_UNTOKENIZED_FIELD); List> result = collectBySearchGoods(keyword, keywordFields, false, collectedField, filters); if (CollectionUtils.isEmpty(result)) { result = collectBySearchGoods(keyword, null, true, collectedField, filters); } return result; } /** * @param keyword * @param keywordFields * 要查询的字段 * @param tokenized * 是否分词 * @param collectedField * @param filters * @return */ private List> collectBySearchGoods(String keyword, List keywordFields, Boolean tokenized, CollectField collectedField, Map filters) { if (collectedField == null && CollectionUtils.isEmpty(filters)) { throw new SearchException("参数不合法:collectedField=" + collectedField + ", filter=" + filters); } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(SearchConstants.GOODS_TABLE_NAME); List> result = new ArrayList<>(); try { BooleanQuery booleanQuery = queryGoods(keyword, keywordFields, tokenized); // 过滤 Set> entrySet = filters.entrySet(); for (Entry entry : entrySet) { switch (entry.getKey()) { case GOODS_KINDID: filter(entry.getValue(), SearchConstants.GOODS_KI_ID_FIELD, booleanQuery); break; case GOODS_BRANDID: filter(entry.getValue(), SearchConstants.GOODS_BR_ID_FIELD, booleanQuery); break; case GOODS_STORE_TYPE: filter(entry.getValue(), SearchConstants.GOODS_ST_TYPE_FIELD, booleanQuery); break; case GOODS_CRNAME: filter(entry.getValue(), SearchConstants.GOODS_CRNAME_FIELD, booleanQuery); break; default: throw new SearchException("不支持该过滤字段:" + entry.getKey()); } } logger.info(booleanQuery.toString()); // 统计 String uniqueField; Set fieldsToLoad = new HashSet<>(); switch (collectedField) { case GOODS_KIND: uniqueField = SearchConstants.GOODS_KI_ID_FIELD; fieldsToLoad.add(SearchConstants.GOODS_KI_ID_FIELD); fieldsToLoad.add(SearchConstants.GOODS_KI_NAME_CN_FIELD); break; case GOODS_BRAND: uniqueField = SearchConstants.GOODS_BR_ID_FIELD; fieldsToLoad.add(SearchConstants.GOODS_BR_ID_FIELD); fieldsToLoad.add(SearchConstants.GOODS_BR_UUID_FIELD); fieldsToLoad.add(SearchConstants.GOODS_BR_NAME_CN_FIELD); fieldsToLoad.add(SearchConstants.GOODS_BR_NAME_EN_FIELD); break; case GOODS_STORE_TYPE: uniqueField = SearchConstants.GOODS_ST_TYPE_FIELD; fieldsToLoad.add(SearchConstants.GOODS_ST_TYPE_FIELD); break; case GOODS_CRNAME: uniqueField = SearchConstants.GOODS_CRNAME_FIELD; fieldsToLoad.add(SearchConstants.GOODS_CRNAME_FIELD); break; default: throw new SearchException("不支持该统计字段:" + collectedField); } GoodsGroupCollector collector = new GoodsGroupCollector(uniqueField, fieldsToLoad); indexSearcher.search(booleanQuery, collector); result = collector.getValues(); } catch (IOException e) { logger.error("", e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return result; } /** * 获取查询批次的query * * @param keyword * @param keywordFields * @param tokenized * @return */ private BooleanQuery queryGoods(String keyword, List keywordFields, Boolean tokenized) { BooleanQuery booleanQuery = new BooleanQuery(); if (!SearchUtils.isKeywordInvalid(keyword)) { // 未指定搜索的字段,则采用默认搜索逻辑 if (CollectionUtils.isEmpty(keywordFields)) { booleanQuery.add(setGoodsBoost(keyword), BooleanClause.Occur.MUST); } else { BooleanQuery booleanQuery2 = new BooleanQuery(); for (String keywordField : keywordFields) { // 是否分词 if (tokenized == null || !tokenized.booleanValue()) { booleanQuery2.add(new TermQuery(new Term(keywordField, keyword)), BooleanClause.Occur.SHOULD); } else { booleanQuery2.add(SearchUtils.getBooleanQuery(keywordField, keyword), BooleanClause.Occur.SHOULD); } } booleanQuery.add(booleanQuery2, BooleanClause.Occur.MUST); } } return booleanQuery; } /** * 同时搜索器件、类目、品牌,并设置boost * * @param keyword * @return */ private Query setGoodsBoost(String keyword) { BooleanQuery booleanQuery = new BooleanQuery(); BooleanQuery componentCodeQuery = SearchUtils.getBooleanQuery(SearchConstants.GOODS_CMP_CODE_FIELD, keyword); componentCodeQuery.setBoost(100); booleanQuery.add(componentCodeQuery, BooleanClause.Occur.SHOULD); BooleanQuery brandNameCnQuery = SearchUtils.getBooleanQuery(SearchConstants.GOODS_BR_NAME_CN_FIELD, keyword); brandNameCnQuery.setBoost(10); booleanQuery.add(brandNameCnQuery, BooleanClause.Occur.SHOULD); BooleanQuery brandNameEnQuery = SearchUtils.getBooleanQuery(SearchConstants.GOODS_BR_NAME_EN_FIELD, keyword); brandNameEnQuery.setBoost(10); booleanQuery.add(brandNameEnQuery, BooleanClause.Occur.SHOULD); BooleanQuery kindNameQuery = SearchUtils.getBooleanQuery(SearchConstants.GOODS_KI_NAME_CN_FIELD, keyword); kindNameQuery.setBoost(1); booleanQuery.add(kindNameQuery, BooleanClause.Occur.SHOULD); return booleanQuery; } /** * 过滤 * * @param list * 过滤值列表 * @param field * 过滤的字段 * @param booleanQuery * 查询条件 */ @SuppressWarnings("unchecked") private void filter(Object list, String field, BooleanQuery booleanQuery) { List values; if (list instanceof List) { values = (List) list; }else{ values = new ArrayList<>(); values.add(list); } BooleanQuery booleanQuery2 = new BooleanQuery(); for (Object value : values) { TermQuery query = new TermQuery(new Term(field, value.toString().toLowerCase())); booleanQuery2.add(query, BooleanClause.Occur.SHOULD); } booleanQuery.add(booleanQuery2, BooleanClause.Occur.FILTER); } @Override public Kind getKind(Long id) { return DocumentToObjectUtils.toKind( SearchUtils.getDocumentById(SearchConstants.KIND_TABLE_NAME, SearchConstants.KIND_ID_FIELD, id)); } @Override public Brand getBrand(Long id) { return DocumentToObjectUtils.toBrand( SearchUtils.getDocumentById(SearchConstants.BRAND_TABLE_NAME, SearchConstants.BRAND_ID_FIELD, id)); } @Override public Component getComponent(Long id) { return DocumentToObjectUtils.toComponent(SearchUtils.getDocumentById(SearchConstants.COMPONENT_TABLE_NAME, SearchConstants.COMPONENT_ID_FIELD, id)); } @Override public Goods getGoods(String id) { return DocumentToObjectUtils.toGoods( SearchUtils.getDocumentById(SearchConstants.GOODS_TABLE_NAME, SearchConstants.GOODS_ID_FIELD, id)); } @Override public SPage getObjects(String tableName, String keyword, String field, Boolean tokenized, @NotEmpty("page") Integer page, @NotEmpty("size") Integer size) { if (keyword == null) { keyword = ""; } if (field == null) { field = SearchUtils.getIdField(tableName); } if (tokenized == null) { tokenized = false; } IndexSearcher indexSearcher = SearchUtils.getIndexSearcher(tableName); SPage sPage = new SPage<>(); try { Query query; if (tokenized) { query = SearchUtils.getBooleanQuery(field, keyword); } else { query = SearchUtils.getRegexpQuery(field, keyword); } // 分页信息 if (page > 0) { sPage.setPage(page); } else { sPage.setPage(1); sPage.setFirst(true); } if (size > 0) { sPage.setSize(size); } else { sPage.setSize(20); } TopDocs topDocs; // 如果页码不为1 if (sPage.getPage() > 1) { TopDocs previousTopDocs = indexSearcher.search(query, (sPage.getPage() - 1) * sPage.getSize()); int totalHits = previousTopDocs.totalHits; ScoreDoc[] previousScoreDocs = previousTopDocs.scoreDocs; if ((sPage.getPage() - 1) * sPage.getSize() >= totalHits) { throw new SearchException("页码过大:元素总数量为" + totalHits); } topDocs = indexSearcher.searchAfter(previousScoreDocs[previousScoreDocs.length - 1], query, sPage.getSize()); } else { sPage.setFirst(true); topDocs = indexSearcher.search(query, sPage.getSize()); } int totalHits = topDocs.totalHits; // 设置总元素个数、页数等信息 sPage.setTotalElement(totalHits); int totalPage = (int) Math.ceil(totalHits / (1.0 * sPage.getSize())); sPage.setTotalPage(totalPage); if (totalPage == sPage.getPage()) { sPage.setLast(true); } List content = new ArrayList<>(); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { Document document = indexSearcher.doc(scoreDoc.doc); content.add(DocumentToObjectUtils.toObject(document, tableName)); } sPage.setContent(content); } catch (IOException e) { throw new SearchException(e).setDetailedMessage(e); } finally { SearchUtils.releaseIndexSearcher(indexSearcher); } return sPage; } }