manage/uuspider/init_data/task_manager.py

# coding=utf-8
'''
Created on 2016年3月7日

@author: ChenHao
'''
import random
from mouser_spider import Status
from pymongo.mongo_client import MongoClient


class TaskManager(object):
    
    def __init__(self):
        self.result = None
    
    # 获得一个任务
    def _get_one_task(self, db):
        
        rand = random.random()
        result_gt = db.kindlist_test.find_one({"status" : Status.TODO, "random" : {"$gt" : rand}})
        
        if result_gt is not None:
#             result_gt["status"] = Status.BUSY
            self.result = result_gt
        else:
            result_lt = db.kindlist_test.find_one({"status" : Status.TODO, "random" : {"$lt" : rand}})
            if result_lt is not None:
#                 result_lt["status"] = Status.BUSY
                self.result = result_lt
            
        return self.result
        
if __name__ == '__main__':
    task_manager = TaskManager()
    cli = MongoClient("mongodb://localhost:27017/")
#     db = cli.spider.kindlist_todo
    db = cli.spider
    
    s = set()
    for i in range(100):
        task_manager.result = None
        result = task_manager._get_one_task(db)
        print(result["url"])
        s.add(result["url"])
    
    print(len(s))