数据中心文件归档处理；多文件归档

pom.xml

@@ -126,6 +126,18 @@
 			<version>1.0.26</version>
 		</dependency>
 
+		<dependency>
+			<groupId>org.samba.jcifs</groupId>
+			<artifactId>jcifs</artifactId>
+			<version>1.3.17</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.poi</groupId>
+			<artifactId>poi</artifactId>
+			<version>3.14</version>
+		</dependency>
+
 	</dependencies>
 
 	<build>
@@ -155,8 +167,8 @@
 
 				<configuration>
 
-					<source>1.7</source>
-					<target>1.7</target>
+					<source>8</source>
+					<target>8</target>
 
 					<compilerArguments>
 						<!-- 打包本地jar包 -->

src/main/java/com/uas/eis/entity/DataChip.java

@@ -0,0 +1,54 @@
+package com.uas.eis.entity;
+
+import java.util.Date;
+
+/**
+ * @Author: zhouy
+ * @Date: 2020/5/26 13:53
+ */
+public class DataChip {
+    private Integer id_;
+    private String chipcode_;
+    private String counterpath_;
+    private String datapath_;
+    private Date testdate_;
+    public Integer getId_() {
+        return id_;
+    }
+
+    public void setId_(Integer id_) {
+        this.id_ = id_;
+    }
+
+    public String getChipcode_() {
+        return chipcode_;
+    }
+
+    public void setChipcode_(String chipcode_) {
+        this.chipcode_ = chipcode_;
+    }
+
+    public String getCounterpath_() {
+        return counterpath_;
+    }
+
+    public void setCounterpath_(String counterpath_) {
+        this.counterpath_ = counterpath_;
+    }
+
+    public String getDatapath_() {
+        return datapath_;
+    }
+
+    public void setDatapath_(String datapath_) {
+        this.datapath_ = datapath_;
+    }
+
+    public Date getTestdate_() {
+        return testdate_;
+    }
+
+    public void setTestdate_(Date testdate_) {
+        this.testdate_ = testdate_;
+    }
+}

src/main/java/com/uas/eis/service/DataDealService.java

@@ -1,5 +1,7 @@
 package com.uas.eis.service;
 
+import com.uas.eis.entity.DataChip;
+
 import java.util.concurrent.CountDownLatch;
 
 /**
@@ -7,5 +9,5 @@ import java.util.concurrent.CountDownLatch;
  * @Date: 2020/5/22 13:54
  */
 public interface DataDealService {
-     void EDCDataDeal(String path, CountDownLatch countDownLatch);
+     void EDCDataDeal(DataChip dataChip, CountDownLatch countDownLatch);
 }

src/main/java/com/uas/eis/serviceImpl/DataDealServiceImpl.java

@@ -1,13 +1,27 @@
 package com.uas.eis.serviceImpl;
 
 import com.uas.eis.dao.BaseDao;
+import com.uas.eis.entity.DataChip;
 import com.uas.eis.service.DataDealService;
+import com.uas.eis.utils.SmbUtil;
+import jcifs.smb.SmbFile;
+import jcifs.smb.SmbFileInputStream;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
+import java.io.BufferedInputStream;
+import java.io.InputStream;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 
 /**
@@ -24,24 +38,103 @@ public class DataDealServiceImpl  implements DataDealService {
 
     @Override
     @Async("taskExecutor")
-    public void EDCDataDeal(String path, CountDownLatch countDownLatch) {
-        logger.info("线程-" + Thread.currentThread().getId() + "在执行写入");
-        int k = 0;
+    public void EDCDataDeal(DataChip dataChip, CountDownLatch countDownLatch) {
         try {
-
-            for (int i = 0; i < 1000000000; i++) {
-                k++;
-                for (int j = 0; j < 10000; j++) {
-                   // k++;
+            //读取文件EXCEL解析
+            String FilePath=dataChip.getDatapath_();
+            Date testdate = dataChip.getTestdate_();
+            String TableName = "RECORD$"+new SimpleDateFormat("yyMMdd").format(testdate);
+            //打开文件流
+            SmbFile smbFile = SmbUtil.getCurrentFile(FilePath);
+            InputStream is = new BufferedInputStream(new SmbFileInputStream(smbFile));
+            Workbook book = new HSSFWorkbook(is);
+            Sheet st=book.getSheet("Data");
+            Row labelRow = st.getRow(13);
+            List<String> items = new ArrayList<>();
+            for(int i=2 ; i <=labelRow.getPhysicalNumberOfCells(); i++){
+                items.add(labelRow.getCell(i).getStringCellValue().replace((i-1)+" ","").trim());
+            }
+            //关键信息解析
+            StringBuffer mainBuffer = new StringBuffer();
+            int mainBeginRow = 14;
+            List<String> itemCols = new ArrayList<>();
+            for(int j = 0 ; j < items.size(); j++ ){
+                mainBuffer.append("INSERT INTO DATACENTER$CHIP_DATAMAIN(CHIPCODE_,DATE_,ITEM_NAME,DETNO,BIAS1,BIAS2,BIAS3,MINLIMIT,MAXLIMIT,AVG_,STDEV,MIN_,MAX_) VALUES (");
+                mainBuffer.append("'"+dataChip.getChipcode_()+"',SYSDATE,'"+items.get(j)+"',"+(j+1)+",");
+                mainBuffer.append("'"+st.getRow(mainBeginRow).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+1).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+2).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+3).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+4).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+5).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+6).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+7).getCell(j+2).getStringCellValue().trim()+"',");
+                mainBuffer.append("'"+st.getRow(mainBeginRow+8).getCell(j+2).getStringCellValue().trim()+"'); ");
+                itemCols.add("ITEM"+(j+1));
+            }
+            List<String> sqls =new ArrayList<String>();
+            sqls.add("BEGIN "+mainBuffer.toString()+"  END;");
+            int allRow = st.getLastRowNum()-23;//总行数
+            //分批3000行处理
+            int a = allRow / 3000;
+            int b = allRow % 3000;
+            StringBuffer detailBuffer=new StringBuffer();
+            String dynamicCols = String.join(",",itemCols);
+            String ValueSql = "";
+            String cellValue= "";
+            Row dataRow = null;
+            if(a > 0) {
+                for ( int i = 0 ; i < a ; i++){
+                    detailBuffer.setLength(0);
+                    for(int k = i*3000 ; k < (i+1)*3000 ; k++){
+                        detailBuffer.append("INSERT INTO "+TableName+"@SHENAI_EDCDATA (CHIPCODE,DETNO,"+String.join(",",dynamicCols)+") VALUES " +
+                                "('"+dataChip.getChipcode_()+"',"+(k+1));
+                        ValueSql = "";
+                        dataRow = st.getRow(k+24);
+                        for ( int m = 0 ; m < items.size() ; m++) {
+                            cellValue = dataRow.getCell(m+2).getStringCellValue();
+                            ValueSql += cellValue !=null? "'"+cellValue.trim()+"'" : "''";
+                            if(m < items.size()-1 ) {
+                                ValueSql += ",";
+                            }
+                        }
+                        detailBuffer.append(","+ValueSql+");");
+                    }
+                    sqls.add(" BEGIN "+detailBuffer.toString()+"  END;");
+                }
+            }
+            if(b >0 ){
+                detailBuffer.setLength(0);
+                for(int k = 0 ; k < b ; k++){
+                    detailBuffer.append("INSERT INTO "+TableName+"@SHENAI_EDCDATA (CHIPCODE,DETNO,"+String.join(",",dynamicCols)+") VALUES " +
+                            "('"+dataChip.getChipcode_()+"',"+(a*3000+k+1));
+                    ValueSql = "";
+                    dataRow = st.getRow(a*3000+k+24);
+                    for ( int m = 0 ; m < items.size() ; m++) {
+                        cellValue = dataRow.getCell(m+2).getStringCellValue();
+                        ValueSql += cellValue !=null? "'"+cellValue.trim()+"'" : "''";
+                        if(m < items.size()-1 ) {
+                            ValueSql += ",";
+                        }
+                    }
+                    detailBuffer.append(","+ValueSql+");");
                 }
+               sqls.add("BEGIN "+detailBuffer.toString()+"  END;");
             }
+            //插入归档
+
+            //更新为失效状态
+            sqls.add("UPDATE CHIPTABLE@SHENAI_EDCDATA SET STATUS_=0  WHERE CHIP_CODE='"+dataChip.getChipcode_()+"'");
+            //插入有效
+            sqls.add("INSERT INTO CHIPTABLE@SHENAI_EDCDATA (CHIP_CODE,TABLE_NAME,STATUS_) VALUES ('"+dataChip.getChipcode_()+"','"+TableName+"',-1)");
+            //更新读写状态
+            sqls.add("UPDATE DATACENTER$CHIP SET READSTATUS_=-1 WHERE ID_="+dataChip.getId_());//更新成功执行
+            baseDao.getJdbcTemplate().batchUpdate(sqls.toArray(new String[sqls.size()]));
         }catch (Exception e) {
+            baseDao.execute("UPDATE DATACENTER$CHIP SET READSTATUS_=-99 WHERE ID_="+dataChip.getId_());
             e.printStackTrace();
-
         } finally {
             countDownLatch.countDown();
         }
-        System.out.println("线程" + Thread.currentThread().getId() + "执行结束:"+k);
-
     }
 }

src/main/java/com/uas/eis/task/DataDealTask.java

@@ -0,0 +1,53 @@
+package com.uas.eis.task;
+
+import com.uas.eis.dao.BaseDao;
+import com.uas.eis.entity.DataChip;
+import com.uas.eis.service.DataDealService;
+import com.uas.eis.service.SaleReturnService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.BeanPropertyRowMapper;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
+
+/**
+ * @Author: zhouy
+ * @Date: 2020/5/26 18:51
+ */
+@Component
+public class DataDealTask {
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+    @Autowired
+    BaseDao baseDao;
+    @Autowired
+    DataDealService dataDealService;
+
+    @Scheduled(cron = "0 0/10 * * * ?")
+    public void dataDeal() {
+        List<DataChip> dataChips = baseDao.getJdbcTemplate().query("select id_,chipcode_,counterpath_,datapath_,testdate_ from (select id_,chipcode_,counterpath_,datapath_,testdate_ from datacenter$chip where  nvl(readstatus_,0)=0 and testdate_ is  not null order by testdate_  desc) where rownum<=50",
+                new BeanPropertyRowMapper<DataChip>(DataChip.class));
+        logger.info("Data文件解析执行开始：文件数"+dataChips.size());
+        Date date = new Date();
+        if(dataChips.size()>0){
+            List<Integer> IDS = dataChips.stream().map(DataChip::getId_).collect(Collectors.toList());
+            final CountDownLatch countDownLatch = new CountDownLatch(IDS.size());
+            baseDao.execute("UPDATE DATACENTER$CHIP SET READSTATUS_=-90 WHERE ID_ IN ("+String.join(",", IDS.stream().map(String::valueOf).collect(Collectors.toList()))+")");
+            try {
+                dataChips.stream().forEach(dataChip -> {
+                    dataDealService.EDCDataDeal(dataChip,countDownLatch);
+                });
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+        logger.info("Data文件解析执行结束:用时"+((date.getTime()-new Date().getTime())/1000));
+    }
+
+}

src/main/java/com/uas/eis/task/SaleReturnTask.java

@@ -1,21 +0,0 @@
-package com.uas.eis.task;
-
-import com.uas.eis.service.SaleReturnService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-/**
- * Created by luhg on 2018/4/26.
- * 销售退货单
- */
-@Component
-public class SaleReturnTask {
-    @Autowired
-    SaleReturnService saleReturnService;
-
-    @Scheduled(cron = "0 0 0/1 * * ?")
-    public void getSaleReturn() {
-        saleReturnService.getSaleReturn();
-    }
-}

src/main/java/com/uas/eis/task/SaleTask.java

@@ -1,22 +0,0 @@
-package com.uas.eis.task;
-
-import com.uas.eis.service.SaleService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-/**
- * Created by luhg on 2018/4/26.
- * 销售订单
- */
-@Component
-public class SaleTask {
-    @Autowired
-    SaleService saleService;
-
-
-    @Scheduled(cron = "0 0/55 * * * ?")
-    public void getSale() {
-        saleService.getSale();
-    }
-}

src/main/java/com/uas/eis/task/StockTask.java

@@ -1,21 +0,0 @@
-package com.uas.eis.task;
-
-import com.uas.eis.service.StockService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-/**
- * Created by luhg on 2018/4/24.
- * 出入库定时任务
- */
-@Component
-public class StockTask {
-    @Autowired
-    StockService stockService;
-
-    @Scheduled(cron = "0 0/50 * * * ?")
-    public void autoPost() {
-        stockService.autoPost(); //出入库定时过账
-    }
-}

src/main/java/com/uas/eis/utils/SmbUtil.java

@@ -0,0 +1,111 @@
+package com.uas.eis.utils;
+
+import jcifs.smb.NtlmPasswordAuthentication;
+import jcifs.smb.SmbFile;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author: zhouy
+ * @Date: 2020/3/30 14:58
+ */
+public class SmbUtil {
+    private static String user = "administrator"; //用户名
+    private static String pwd = "0DlbAdmin0"; //密码
+    public static String centerIP = "172.16.0.15";
+    public static String centerPath = "D$/";
+
+
+    /**
+     * 获取远程访问地址
+     */
+    public static String getSmbRootUrl(String serverID, String rootPath){
+        return  String.format("smb://@%s%s%s",serverID,"/",rootPath);
+    }
+
+    public static NtlmPasswordAuthentication getAuth(){
+        return new NtlmPasswordAuthentication(null,user,pwd);
+    }
+
+    public static SmbFile getRootFile(String serverID, String rootPath){
+        SmbFile smbFile = null;
+        try {
+            smbFile = new SmbFile(getSmbRootUrl(serverID,rootPath),getAuth());
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        }
+        return smbFile;
+
+    }
+
+    public static SmbFile getCurrentFile(String serverID, String path){
+        SmbFile smbFile = null;
+        try {
+            smbFile = new SmbFile(getSmbRootUrl(serverID,path),getAuth());
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        }
+        return smbFile;
+    }
+
+    public static SmbFile getCurrentFile(String path){
+        SmbFile smbFile = null;
+        try {
+            smbFile = new SmbFile(path,getAuth());
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        }
+        return smbFile;
+
+    }
+
+    public static List<String> getChildFiles(SmbFile smbFile,String startChar){
+        List<String> paths = new ArrayList<>();
+        SmbFile file = null;
+        try {
+            smbFile.connect();
+            if(smbFile.isDirectory()){
+                SmbFile[] files = smbFile.listFiles();
+                for (int i = 0; i<files.length; i++){
+                    file = files[i];
+                    if(file.isFile()){
+                        if(file.getName().startsWith(startChar)){
+                            paths.add(files[i].getPath());
+                        }
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return paths;
+    }
+
+    public static List<SmbFile> getChildFiles(SmbFile smbFile,String startChar, String docType){
+        List<SmbFile> files = new ArrayList<>();
+        SmbFile childFile = null;
+        String fileName = null;
+        try {
+            smbFile.connect();
+            if(smbFile.isDirectory()){
+                SmbFile [] childs = smbFile.listFiles();
+                for (int i = 0; i<childs.length; i++){
+                    childFile = childs[i];
+                    if(childFile.isFile()){
+                        fileName = childFile.getName();
+                        if(fileName.toUpperCase().startsWith(startChar) && fileName.endsWith(docType)) {
+                            files.add(childFile);
+                        }
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return files;
+    }
+}
+

src/main/resources/application.yml

@@ -2,10 +2,9 @@ spring:
     datasource:
         type: com.alibaba.druid.pool.DruidDataSource
         driverClassName: oracle.jdbc.OracleDriver
-        username: UAS_DEV
+        username: SZSI_P
         password: select!#%*(
-#        url: jdbc:oracle:thin:@oclean.uicp.net:1521:orcl
-        url: jdbc:oracle:thin:@218.17.253.75:11521:orcl
+        url: jdbc:oracle:thin:@172.16.0.22:1521:orcl
         initialSize: 10
         maxActive: 80
         minIdle: 10

src/test/java/com/uas/eis/UasEisApplicationTests.java

@@ -1,5 +1,7 @@
 package com.uas.eis;
 
+import com.uas.eis.dao.BaseDao;
+import com.uas.eis.entity.DataChip;
 import com.uas.eis.service.DataDealService;
 import com.uas.eis.service.SaleReturnService;
 import com.uas.eis.service.SaleService;
@@ -7,41 +9,42 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.jdbc.core.BeanPropertyRowMapper;
+import org.springframework.jdbc.support.rowset.SqlRowSet;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = {UasEisApplication.class})
 public class UasEisApplicationTests {
 
-	@Autowired
-	private SaleService SaleService;
-	@Autowired
-	private SaleReturnService SaleReturnService;
 	@Autowired
 	private DataDealService dataDealService;
-
-
-
-
+	@Autowired
+	private BaseDao baseDao;
 	@Test
 	public void contextLoads() {
-		final CountDownLatch countDownLatch = new CountDownLatch(10);
-		Date date1 =new Date();
-		System.out.println("主线程开始---");
-
-		try {
-			for(int i =0 ; i<10 ;i++){
-				dataDealService.EDCDataDeal("helo"+i,countDownLatch);
+		List<DataChip> dataChips = baseDao.getJdbcTemplate().query("select id_,chipcode_,counterpath_,datapath_,testdate_ from (select id_,chipcode_,counterpath_,datapath_,testdate_ from datacenter$chip where chipcode_='F20025075#04' and  nvl(readstatus_,0)=0 and testdate_ is  not null order by testdate_  desc) where rownum<=50",
+				new BeanPropertyRowMapper<DataChip>(DataChip.class));
+		if(dataChips.size()>0){
+			List<Integer> IDS = dataChips.stream().map(DataChip::getId_).collect(Collectors.toList());
+			final CountDownLatch countDownLatch = new CountDownLatch(IDS.size());
+			baseDao.execute("UPDATE DATACENTER$CHIP SET READSTATUS_=-90 WHERE ID_ IN ("+String.join(",", IDS.stream().map(String::valueOf).collect(Collectors.toList()))+")");
+			try {
+				dataChips.stream().forEach(dataChip -> {
+					dataDealService.EDCDataDeal(dataChip,countDownLatch);
+				});
+				countDownLatch.await();
+			} catch (InterruptedException e) {
+				e.printStackTrace();
 			}
-			countDownLatch.await();
-		} catch (InterruptedException e) {
-			e.printStackTrace();
 		}
-		System.out.println("用时:"+(new Date().getTime()-date1.getTime()));
-
 	}
 
+
 }