| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869 |
- # coding=utf-8
- '''
- Created on 2016年3月7日
- 为详情页生成下载任务
- @author: ChenHao
- '''
- from pymongo.mongo_client import MongoClient
- from util_common import Constant, html_parser
- from bs4 import BeautifulSoup
- import random
- praser = html_parser.HtmlParser()
- # 连接数据库
- cli = MongoClient(Constant.MONGODB_URL)
- db = cli.spider
- # 读取kindlist待生成detail任务的
- new_product_urls = set()
- rs = db.kindlist_todo.find({"status": Constant.DONE ,"creatDetailTask": Constant.TODO}, no_cursor_timeout=True)
- # listPage = rs;
- for listPage in rs:
- # 生成任务并保存
- url_listPage = listPage["url"]
- html_cont = listPage["str_html"]
- # 检查列表页的页面大小是不是正常，例如一个的大小为332004
- if len(html_cont) < 150000:
- db.kindlist_todo.update_one({'url':url_listPage}, {'$set': {'creatDetailTask': Constant.ERROR}})
- continue
- try:
- soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
- # 解析并存储解析出来的类目
- kindls = praser._get_kindlist_by_listPage(soup)
- kind_d = dict()
- kind_d["kindls"] = kindls
- db.kind_from_listpage.insert_one(kind_d)
- # 解析并存储detailTask
- detail_urls = praser._get_detail_urls_from_listPage(soup)
- # 组装detail任务
- task_list = list()
- for detail_url in detail_urls:
- d = dict()
- d["url"] = detail_url
- d["random"] = random.random()
- d["status"] = Constant.TODO
- d["analysisTask"] = Constant.TODO
- task_list.append(d)
- db.detail_todo.insert_many(task_list)
- # 将此listPage修改状态
- db.kindlist_todo.update_one({'url':url_listPage}, {'$set': {'creatDetailTask': Constant.DONE}})
- except:
- # 将此listPage修改为错误状态
- db.kindlist_todo.update_one({'url':url_listPage}, {'$set': {'creatDetailTask': Constant.ERROR}})
-
- cli.close()
|
