|
@@ -6,7 +6,6 @@ import java.io.FileNotFoundException;
|
|
|
import java.io.FileReader;
|
|
import java.io.FileReader;
|
|
|
import java.io.IOException;
|
|
import java.io.IOException;
|
|
|
import java.io.PrintWriter;
|
|
import java.io.PrintWriter;
|
|
|
-import java.nio.file.Paths;
|
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.ArrayList;
|
|
|
import java.util.Date;
|
|
import java.util.Date;
|
|
|
import java.util.List;
|
|
import java.util.List;
|
|
@@ -17,7 +16,6 @@ import org.apache.log4j.Logger;
|
|
|
import org.apache.lucene.document.Document;
|
|
import org.apache.lucene.document.Document;
|
|
|
import org.apache.lucene.index.IndexWriter;
|
|
import org.apache.lucene.index.IndexWriter;
|
|
|
import org.apache.lucene.index.Term;
|
|
import org.apache.lucene.index.Term;
|
|
|
-import org.apache.lucene.store.FSDirectory;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.data.domain.Page;
|
|
import org.springframework.data.domain.Page;
|
|
|
import org.springframework.data.jpa.repository.JpaRepository;
|
|
import org.springframework.data.jpa.repository.JpaRepository;
|
|
@@ -46,79 +44,54 @@ import com.uas.search.console.b2b.util.SearchUtils;
|
|
|
@Service
|
|
@Service
|
|
|
public class IndexServiceImpl implements IndexService {
|
|
public class IndexServiceImpl implements IndexService {
|
|
|
|
|
|
|
|
|
|
+ @Autowired
|
|
|
|
|
+ private AQListener aqListener;
|
|
|
|
|
+
|
|
|
private static final int PAGE_SIZE = 1000;
|
|
private static final int PAGE_SIZE = 1000;
|
|
|
|
|
|
|
|
// 单个文件存储的最大数据数目
|
|
// 单个文件存储的最大数据数目
|
|
|
public static final int SINGLE_FILE_MAX_SIZE = 100000;
|
|
public static final int SINGLE_FILE_MAX_SIZE = 100000;
|
|
|
|
|
|
|
|
- private IndexWriter indexWriter;
|
|
|
|
|
-
|
|
|
|
|
- private static IndexWriterManager indexWriterManager;
|
|
|
|
|
-
|
|
|
|
|
- private FSDirectory directory;
|
|
|
|
|
-
|
|
|
|
|
- @Autowired
|
|
|
|
|
- private AQListener aqListener;
|
|
|
|
|
-
|
|
|
|
|
/**
|
|
/**
|
|
|
* 是否正在创建索引
|
|
* 是否正在创建索引
|
|
|
*/
|
|
*/
|
|
|
private boolean creatingIndex = false;
|
|
private boolean creatingIndex = false;
|
|
|
|
|
|
|
|
- Logger logger = Logger.getLogger(getClass());
|
|
|
|
|
|
|
+ private IndexWriter indexWriter;
|
|
|
|
|
|
|
|
- public IndexServiceImpl() {
|
|
|
|
|
- try {
|
|
|
|
|
- directory = FSDirectory.open(Paths.get(SearchConstants.INDEX_DIR));
|
|
|
|
|
- indexWriterManager = new IndexWriterManager(directory);
|
|
|
|
|
- } catch (IOException e) {
|
|
|
|
|
- e.printStackTrace();
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ private static IndexWriterManager indexWriterManager = new IndexWriterManager();
|
|
|
|
|
+
|
|
|
|
|
+ Logger logger = Logger.getLogger(getClass());
|
|
|
|
|
|
|
|
@Override
|
|
@Override
|
|
|
- public Long createIndexs(Boolean fromFiles) {
|
|
|
|
|
|
|
+ public Long createIndexs(List<Table_name> tableNames, Boolean fromFiles) {
|
|
|
if (creatingIndex) {
|
|
if (creatingIndex) {
|
|
|
throw new SearchException("已存在线程在创建索引,不可重复请求");
|
|
throw new SearchException("已存在线程在创建索引,不可重复请求");
|
|
|
}
|
|
}
|
|
|
creatingIndex = true;
|
|
creatingIndex = true;
|
|
|
- // 如果索引实时更新处于开启状态,需要暂时关闭(以免两者同时操作索引出现问题)
|
|
|
|
|
|
|
+ // 如果索引实时更新处于开启状态,需要关闭(以免两者同时操作索引出现问题)
|
|
|
if (aqListener.isRunning()) {
|
|
if (aqListener.isRunning()) {
|
|
|
logger.info("索引实时更新服务正在运行,尝试关闭索引实时更新服务...");
|
|
logger.info("索引实时更新服务正在运行,尝试关闭索引实时更新服务...");
|
|
|
aqListener.stop();
|
|
aqListener.stop();
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- try {
|
|
|
|
|
- indexWriter = indexWriterManager.get();
|
|
|
|
|
- logger.info("正在清理旧索引...");
|
|
|
|
|
- indexWriter.deleteAll();
|
|
|
|
|
- indexWriter.commit();
|
|
|
|
|
- logger.info("旧索引清理完毕");
|
|
|
|
|
- Long startTime = new Date().getTime();
|
|
|
|
|
-
|
|
|
|
|
- // 创建各个实体类对应表的索引
|
|
|
|
|
- List<Class<?>> clazzes = ClassAndTableNameUtils.getEntityClasses();
|
|
|
|
|
- for (Class<?> clazz : clazzes) {
|
|
|
|
|
- // 只有明确指定不从本地文件读取数据时,才从数据库获取数据建立索引
|
|
|
|
|
- if (fromFiles != null && !fromFiles) {
|
|
|
|
|
- createIndexes(clazz);
|
|
|
|
|
- } else {
|
|
|
|
|
- createIndexesFromFiles(clazz);
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ if (CollectionUtils.isEmpty(tableNames)) {
|
|
|
|
|
+ tableNames = ClassAndTableNameUtils.getTableNames();
|
|
|
|
|
+ }
|
|
|
|
|
+ Long startTime = new Date().getTime();
|
|
|
|
|
+ // 创建各个表的索引
|
|
|
|
|
+ for (Table_name tableName : tableNames) {
|
|
|
|
|
+ // 只有明确指定不从本地文件读取数据时,才从数据库获取数据建立索引
|
|
|
|
|
+ if (fromFiles != null && !fromFiles) {
|
|
|
|
|
+ createIndexes(ClassAndTableNameUtils.toClass(tableName));
|
|
|
|
|
+ } else {
|
|
|
|
|
+ createIndexesFromFiles(ClassAndTableNameUtils.toClass(tableName));
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
- Long endTime = new Date().getTime();
|
|
|
|
|
- logger.info("索引创建成功, 共用时间 " + (endTime - startTime) + " ms");
|
|
|
|
|
- return endTime - startTime;
|
|
|
|
|
- } catch (IOException e) {
|
|
|
|
|
- e.printStackTrace();
|
|
|
|
|
- } catch (InterruptedException e) {
|
|
|
|
|
- e.printStackTrace();
|
|
|
|
|
- } finally {
|
|
|
|
|
- creatingIndex = false;
|
|
|
|
|
- indexWriterManager.release();
|
|
|
|
|
}
|
|
}
|
|
|
- return null;
|
|
|
|
|
|
|
+ creatingIndex = false;
|
|
|
|
|
+ Long endTime = new Date().getTime();
|
|
|
|
|
+ logger.info(String.format("索引创建成功, 共用时间%.2fs\n", (endTime - startTime) / 1000.0));
|
|
|
|
|
+ return endTime - startTime;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -128,57 +101,66 @@ public class IndexServiceImpl implements IndexService {
|
|
|
* @return 实际创建的索引的数目
|
|
* @return 实际创建的索引的数目
|
|
|
* @throws IOException
|
|
* @throws IOException
|
|
|
*/
|
|
*/
|
|
|
- private <T> Long createIndexes(Class<T> clazz) throws IOException {
|
|
|
|
|
|
|
+ private <T> Long createIndexes(Class<T> clazz) {
|
|
|
Long startTime = new Date().getTime();
|
|
Long startTime = new Date().getTime();
|
|
|
Table_name tableName = ClassAndTableNameUtils.toTableName(clazz);
|
|
Table_name tableName = ClassAndTableNameUtils.toTableName(clazz);
|
|
|
logger.info("正在创建" + tableName.value() + "索引...");
|
|
logger.info("正在创建" + tableName.value() + "索引...");
|
|
|
JpaRepository<T, Long> dao = ClassAndTableNameUtils.getDao(clazz);
|
|
JpaRepository<T, Long> dao = ClassAndTableNameUtils.getDao(clazz);
|
|
|
|
|
|
|
|
- // 分页获取数据以创建索引
|
|
|
|
|
- PageParams pageParams = new PageParams();
|
|
|
|
|
- pageParams.setPage(1);
|
|
|
|
|
- pageParams.setCount(PAGE_SIZE);
|
|
|
|
|
- PageInfo pageInfo = new PageInfo(pageParams);
|
|
|
|
|
- Page<T> pageResult = dao.findAll(pageInfo);
|
|
|
|
|
-
|
|
|
|
|
- // 数据库中数据的总数目
|
|
|
|
|
- long totalElements = pageResult.getTotalElements();
|
|
|
|
|
- logger.info("发现数据:" + totalElements + "条");
|
|
|
|
|
- // 已翻页的数据数目
|
|
|
|
|
- Long size = 0L;
|
|
|
|
|
// 实际创建的索引数目(可能会有部分数据有问题,不会创建索引)
|
|
// 实际创建的索引数目(可能会有部分数据有问题,不会创建索引)
|
|
|
long count = 0;
|
|
long count = 0;
|
|
|
- // 用于记录上次提交索引时的创建进度
|
|
|
|
|
- double recordProgress = 0;
|
|
|
|
|
-
|
|
|
|
|
- while (totalElements > size) {
|
|
|
|
|
- List<T> content = pageResult.getContent();
|
|
|
|
|
- for (T element : content) {
|
|
|
|
|
- Document document = ObjectToDocumentUtils.toDocument(element);
|
|
|
|
|
- if (document != null) {
|
|
|
|
|
- indexWriter.addDocument(document);
|
|
|
|
|
- count++;
|
|
|
|
|
|
|
+ // 数据库中数据的总数目
|
|
|
|
|
+ long totalElements = 0L;
|
|
|
|
|
+ try {
|
|
|
|
|
+ deleteIndexs(tableName);
|
|
|
|
|
+ // 分页获取数据以创建索引
|
|
|
|
|
+ PageParams pageParams = new PageParams();
|
|
|
|
|
+ pageParams.setPage(1);
|
|
|
|
|
+ pageParams.setCount(PAGE_SIZE);
|
|
|
|
|
+ PageInfo pageInfo = new PageInfo(pageParams);
|
|
|
|
|
+ Page<T> pageResult = dao.findAll(pageInfo);
|
|
|
|
|
+
|
|
|
|
|
+ totalElements = pageResult.getTotalElements();
|
|
|
|
|
+ logger.info("发现数据:" + totalElements + "条");
|
|
|
|
|
+ // 已翻页的数据数目
|
|
|
|
|
+ Long size = 0L;
|
|
|
|
|
+ // 用于记录上次提交索引时的创建进度
|
|
|
|
|
+ double recordProgress = 0;
|
|
|
|
|
+
|
|
|
|
|
+ while (totalElements > size) {
|
|
|
|
|
+ List<T> content = pageResult.getContent();
|
|
|
|
|
+ for (T element : content) {
|
|
|
|
|
+ Document document = ObjectToDocumentUtils.toDocument(element);
|
|
|
|
|
+ if (document != null) {
|
|
|
|
|
+ indexWriter.addDocument(document);
|
|
|
|
|
+ count++;
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
- }
|
|
|
|
|
- size += content.size();
|
|
|
|
|
|
|
+ size += content.size();
|
|
|
|
|
|
|
|
- // 器件索引的创建进度(百分比)
|
|
|
|
|
- double indexProgress = size * 100.0 / totalElements;
|
|
|
|
|
- logger.info(String.format("Created...................%.2f%%", indexProgress));
|
|
|
|
|
- // 每创建5%,提交一次,避免内存耗尽,发生OutOfMemoryError
|
|
|
|
|
- if (indexProgress - recordProgress > 5) {
|
|
|
|
|
- indexWriter.commit();
|
|
|
|
|
- recordProgress = indexProgress;
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ // 器件索引的创建进度(百分比)
|
|
|
|
|
+ double indexProgress = size * 100.0 / totalElements;
|
|
|
|
|
+ logger.info(String.format("Created...................%.2f%%", indexProgress));
|
|
|
|
|
+ // 每创建5%,提交一次,避免内存耗尽,发生OutOfMemoryError
|
|
|
|
|
+ if (indexProgress - recordProgress > 5) {
|
|
|
|
|
+ indexWriter.commit();
|
|
|
|
|
+ recordProgress = indexProgress;
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- pageParams.setPage(pageParams.getPage() + 1);
|
|
|
|
|
- pageInfo = new PageInfo(pageParams);
|
|
|
|
|
- pageResult = dao.findAll(pageInfo);
|
|
|
|
|
|
|
+ pageParams.setPage(pageParams.getPage() + 1);
|
|
|
|
|
+ pageInfo = new PageInfo(pageParams);
|
|
|
|
|
+ pageResult = dao.findAll(pageInfo);
|
|
|
|
|
+ }
|
|
|
|
|
+ indexWriter.commit();
|
|
|
|
|
+ } catch (InterruptedException | IOException e) {
|
|
|
|
|
+ e.printStackTrace();
|
|
|
|
|
+ } finally {
|
|
|
|
|
+ indexWriterManager.release(tableName);
|
|
|
}
|
|
}
|
|
|
- indexWriter.commit();
|
|
|
|
|
|
|
|
|
|
Long endTime = new Date().getTime();
|
|
Long endTime = new Date().getTime();
|
|
|
- logger.info("创建" + tableName.value() + "索引: " + count + "条,耗时 " + (endTime - startTime) + " ms\n");
|
|
|
|
|
|
|
+ logger.info(String.format("创建%s索引:%s条,问题数据:%s条,耗时%.2fs\n", tableName.value(), count, (totalElements - count),
|
|
|
|
|
+ (endTime - startTime) / 1000.0));
|
|
|
return count;
|
|
return count;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -189,7 +171,7 @@ public class IndexServiceImpl implements IndexService {
|
|
|
* @return 实际创建的索引的数目
|
|
* @return 实际创建的索引的数目
|
|
|
* @throws IOException
|
|
* @throws IOException
|
|
|
*/
|
|
*/
|
|
|
- private <T> Long createIndexesFromFiles(Class<T> clazz) throws IOException {
|
|
|
|
|
|
|
+ private <T> Long createIndexesFromFiles(Class<T> clazz) {
|
|
|
Table_name tableName = ClassAndTableNameUtils.toTableName(clazz);
|
|
Table_name tableName = ClassAndTableNameUtils.toTableName(clazz);
|
|
|
logger.info("正在创建" + tableName.value() + "索引...");
|
|
logger.info("正在创建" + tableName.value() + "索引...");
|
|
|
// 数据的总数目
|
|
// 数据的总数目
|
|
@@ -200,9 +182,10 @@ public class IndexServiceImpl implements IndexService {
|
|
|
// 从本地路径读取器件数据
|
|
// 从本地路径读取器件数据
|
|
|
File file = new File(getDataPath(tableName));
|
|
File file = new File(getDataPath(tableName));
|
|
|
if (!file.exists() || ArrayUtils.isEmpty(file.listFiles())) {
|
|
if (!file.exists() || ArrayUtils.isEmpty(file.listFiles())) {
|
|
|
- logger.error("创建索引失败,原因:" + tableName.value() + "数据文件不存在!");
|
|
|
|
|
|
|
+ logger.error("创建索引失败,原因:" + tableName.value() + "数据文件不存在!\n");
|
|
|
return 0L;
|
|
return 0L;
|
|
|
}
|
|
}
|
|
|
|
|
+ deleteIndexs(tableName);
|
|
|
File[] files = file.listFiles();
|
|
File[] files = file.listFiles();
|
|
|
// 将要创建的索引总数目约为:文件数目*单个文件的行数
|
|
// 将要创建的索引总数目约为:文件数目*单个文件的行数
|
|
|
long totalSize = files.length * SINGLE_FILE_MAX_SIZE;
|
|
long totalSize = files.length * SINGLE_FILE_MAX_SIZE;
|
|
@@ -227,13 +210,32 @@ public class IndexServiceImpl implements IndexService {
|
|
|
indexWriter.commit();
|
|
indexWriter.commit();
|
|
|
bufferedReader.close();
|
|
bufferedReader.close();
|
|
|
}
|
|
}
|
|
|
- } catch (IOException e) {
|
|
|
|
|
|
|
+ } catch (IOException | InterruptedException e) {
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
|
|
|
+ } finally {
|
|
|
|
|
+ indexWriterManager.release(tableName);
|
|
|
}
|
|
}
|
|
|
- logger.info(tableName.value() + "数据总数:" + totalElements + "条,创建索引:" + count + "条");
|
|
|
|
|
|
|
+ logger.info(tableName.value() + "数据总数:" + totalElements + "条,创建索引:" + count + "条,问题数据:"
|
|
|
|
|
+ + (totalElements - count) + "条\n");
|
|
|
return count;
|
|
return count;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 删除指定表的索引
|
|
|
|
|
+ *
|
|
|
|
|
+ * @param tableName
|
|
|
|
|
+ * 表名
|
|
|
|
|
+ * @throws InterruptedException
|
|
|
|
|
+ * @throws IOException
|
|
|
|
|
+ */
|
|
|
|
|
+ private void deleteIndexs(Table_name tableName) throws InterruptedException, IOException {
|
|
|
|
|
+ indexWriter = indexWriterManager.get(tableName);
|
|
|
|
|
+ logger.info("正在清理旧索引..." + tableName.value());
|
|
|
|
|
+ indexWriter.deleteAll();
|
|
|
|
|
+ indexWriter.commit();
|
|
|
|
|
+ logger.info("旧索引清理完毕..." + tableName.value());
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
@Override
|
|
@Override
|
|
|
public Long downloadDataFromDatabase(List<Table_name> tableNames) {
|
|
public Long downloadDataFromDatabase(List<Table_name> tableNames) {
|
|
|
Long startTime = new Date().getTime();
|
|
Long startTime = new Date().getTime();
|
|
@@ -334,14 +336,14 @@ public class IndexServiceImpl implements IndexService {
|
|
|
Document document = ObjectToDocumentUtils.toDocument(obj);
|
|
Document document = ObjectToDocumentUtils.toDocument(obj);
|
|
|
if (document != null) {
|
|
if (document != null) {
|
|
|
try {
|
|
try {
|
|
|
- indexWriter = indexWriterManager.get();
|
|
|
|
|
|
|
+ indexWriter = indexWriterManager.get(ClassAndTableNameUtils.toTableName(obj.getClass()));
|
|
|
indexWriter.addDocument(document);
|
|
indexWriter.addDocument(document);
|
|
|
indexWriter.commit();
|
|
indexWriter.commit();
|
|
|
logger.info("Saved... " + obj + "\n");
|
|
logger.info("Saved... " + obj + "\n");
|
|
|
} catch (IOException | InterruptedException e) {
|
|
} catch (IOException | InterruptedException e) {
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
} finally {
|
|
|
- indexWriterManager.release();
|
|
|
|
|
|
|
+ indexWriterManager.release(ClassAndTableNameUtils.toTableName(obj.getClass()));
|
|
|
}
|
|
}
|
|
|
return obj;
|
|
return obj;
|
|
|
} else {
|
|
} else {
|
|
@@ -357,7 +359,7 @@ public class IndexServiceImpl implements IndexService {
|
|
|
Document document = ObjectToDocumentUtils.toDocument(obj);
|
|
Document document = ObjectToDocumentUtils.toDocument(obj);
|
|
|
if (document != null) {
|
|
if (document != null) {
|
|
|
try {
|
|
try {
|
|
|
- indexWriter = indexWriterManager.get();
|
|
|
|
|
|
|
+ indexWriter = indexWriterManager.get(ClassAndTableNameUtils.toTableName(obj.getClass()));
|
|
|
indexWriter.updateDocument(new Term(ClassAndTableNameUtils.getIdField(obj.getClass()),
|
|
indexWriter.updateDocument(new Term(ClassAndTableNameUtils.getIdField(obj.getClass()),
|
|
|
String.valueOf(ClassAndTableNameUtils.getId(obj))), document);
|
|
String.valueOf(ClassAndTableNameUtils.getId(obj))), document);
|
|
|
indexWriter.commit();
|
|
indexWriter.commit();
|
|
@@ -365,7 +367,7 @@ public class IndexServiceImpl implements IndexService {
|
|
|
} catch (IOException | InterruptedException e) {
|
|
} catch (IOException | InterruptedException e) {
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
} finally {
|
|
|
- indexWriterManager.release();
|
|
|
|
|
|
|
+ indexWriterManager.release(ClassAndTableNameUtils.toTableName(obj.getClass()));
|
|
|
}
|
|
}
|
|
|
return obj;
|
|
return obj;
|
|
|
} else {
|
|
} else {
|
|
@@ -381,7 +383,7 @@ public class IndexServiceImpl implements IndexService {
|
|
|
return null;
|
|
return null;
|
|
|
}
|
|
}
|
|
|
try {
|
|
try {
|
|
|
- indexWriter = indexWriterManager.get();
|
|
|
|
|
|
|
+ indexWriter = indexWriterManager.get(ClassAndTableNameUtils.toTableName(obj.getClass()));
|
|
|
indexWriter.deleteDocuments(new Term(ClassAndTableNameUtils.getIdField(obj.getClass()),
|
|
indexWriter.deleteDocuments(new Term(ClassAndTableNameUtils.getIdField(obj.getClass()),
|
|
|
String.valueOf(ClassAndTableNameUtils.getId(obj))));
|
|
String.valueOf(ClassAndTableNameUtils.getId(obj))));
|
|
|
indexWriter.commit();
|
|
indexWriter.commit();
|
|
@@ -390,7 +392,7 @@ public class IndexServiceImpl implements IndexService {
|
|
|
} catch (InterruptedException e) {
|
|
} catch (InterruptedException e) {
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
} finally {
|
|
|
- indexWriterManager.release();
|
|
|
|
|
|
|
+ indexWriterManager.release(ClassAndTableNameUtils.toTableName(obj.getClass()));
|
|
|
}
|
|
}
|
|
|
logger.info("Deleted... " + obj + "\n");
|
|
logger.info("Deleted... " + obj + "\n");
|
|
|
return obj;
|
|
return obj;
|