manage/uuspider/statistics/statistics.py

# coding=utf-8
'''
Created on 2016年3月31日
查看各个任务的统计量
@author: ChenHao
'''
from util_common import Constant
from pymongo.mongo_client import MongoClient
import time

def _append_to_log(fout, cont):
    fout.write(cont + "\n")

fout = open("taskLog.txt", "a")

cli = MongoClient(Constant.MONGODB_URL)
db = cli.spider

timeStamp = time.time()
timeArray = time.localtime(timeStamp)
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)

_append_to_log(fout, otherStyleTime+"\n")
'''
# 查看有多少listPage
'''
rs =  db.kindlist_todo.find()
listpage_all = rs.count()
'''
# 查看有多少listPage没转换完
'''
rs =  db.kindlist_todo.find({"creatDetailTask": Constant.TODO, "status": Constant.DONE})
listpage_toCreatDetail = rs.count()
'''
# 查看有多少listPage转换成功
'''
rs =  db.kindlist_todo.find({"creatDetailTask": Constant.DONE, "status": Constant.DONE})
listpage_CreateSuccess = rs.count()
'''
# 查看有多少listPage转换失败
'''
rs =  db.kindlist_todo.find({"creatDetailTask": Constant.ERROR, "status": Constant.DONE})
listpage_CreateFail = rs.count()  
print ("listPage共计:", listpage_all)
print ("listPage没转换完:", listpage_toCreatDetail)
print ("listPage转换成功:", listpage_CreateSuccess)
print ("listPage转换失败:", listpage_CreateFail)
print ("*****************************")
listPage_cont = "listPage共计:" + str(listpage_all) + "\n" + "listPage没转换完:" + str(listpage_toCreatDetail) + "\n" + "listPage转换成功:" + str(listpage_CreateSuccess) + "\n" + "listPage转换失败:" + str(listpage_CreateFail) + "\n*****************************" 
_append_to_log(fout, listPage_cont)

'''
# 查看有多少detail
'''
rs =  db.detail_todo.find()
detail_all = rs.count()
'''
# 查看有多少detail待下载
'''
rs =  db.detail_todo.find({"status": Constant.TODO})
detail_toDownload = rs.count()
'''
# 查看有多少detail已经下载完,待分析
'''
rs =  db.detail_todo.find({"analysisTask": Constant.TODO, "status": Constant.DONE})
detail_toAnalysis = rs.count()
'''
# 查看有多少detail已经下载完,已分析
'''
rs =  db.detail_todo.find({"analysisTask": Constant.DONE, "status": Constant.DONE})
detail_AnalysisSuccess = rs.count()

print ("detail共计:", detail_all)
print ("detail待下载：", detail_toDownload)
print ("detail已经下载完,待分析:", detail_toAnalysis)
print ("detail已经下载完,已分析：", detail_AnalysisSuccess)
listPage_cont = "detail共计:" + str(detail_all) + "\n" + "detail待下载:" + str(detail_toDownload) + "\n" + "detail已经下载完,待分析:" + str(detail_toAnalysis) + "\n" + "detail已经下载完,已分析:" + str(detail_AnalysisSuccess) + "\n*****************************\n\n\n" 
_append_to_log(fout, listPage_cont)


cli.close()
fout.close()