manage/uuspider/test/pyInstallerTest.py

# coding=utf-8
'''
Created on 2016年3月14日

@author: ChenHao
'''
from util_common import Constant, html_parser
from pymongo.mongo_client import MongoClient
from bs4 import BeautifulSoup
import random

praser = html_parser.HtmlParser()
# 连接数据库
cli = MongoClient(Constant.MONGODB_URL)
db = cli.spider
# 读取kindlist待生成detail任务的

new_product_urls = set()
rs = db.kindlist_todo.find_one({"creatDetailTask": Constant.TODO})
listPage = rs;
# 生成任务并保存
html_cont = listPage["str_html"]
soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
detail_urls = praser._get_detail_urls_from_listPage(soup)
# 组装detail任务
task_list = list()
for detail_url in detail_urls:
    d = dict()
    d["url"] = detail_url
    d["random"] = random.random()
    d["status"] = Constant.TODO
    task_list.append(d)
db.detail_todo.insert_many(task_list)
# 将此listPage修改状态
listPage["creatDetailTask"] = Constant.DONE
db.kindlist_todo.save(listPage)

cli.close()