manage/uuspider/proxy/ip_pool.py

import random
from threading import Timer

import requests
import re


class Pool(object):
    def __init__(self, size=9999, timeout=300):
        # _proxy_/select123@api.ip3366.net
        # 7 days since 2016-3-24 20:17:46
        # 'http://api.ip3366.net/api/?key=20160324194713692&getnum=%s&anonymoustype=3' % size
        
        # 68778130@qq.com/s68778130@http://proxy.mimvp.com/
        # 30 days since 2016-3-28 16:29:50
        self.apis = ['http://proxy.mimvp.com/api/fetch.php?orderid=860160328155901330&num=500', 'http://www.66ip.cn/getzh.php?getzh=2016050519918&getnum=2500&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=0&proxytype=2&api=https'];
        self.proxies = []
        self._refreshing = False
        self._refresh()
        # refresh once every 5 minutes
        timer = Timer(timeout, self._refresh)  
        timer.start()
        
    def _refresh(self):
        if self._refreshing:
            return
        self._refreshing = True
        proxies = []
        for api in self.apis:
            try:
                r = requests.get(api)
                proxies.extend([line.strip() for line in re.split('\n|<br>', r.text) if len(line.strip()) > 0])
            except:
                None
        self.proxies = proxies
        print('init ip pool', len(self.proxies))
        self._refreshing = False
        
    def get(self):
        if len(self.proxies) > 0:
            return random.choice(self.proxies)
        self._refresh()
        return None
    
    def remove(self, value):
        try:
            self.proxies.remove(value)
        except:
            None
        if len(self.proxies) <= 0:
            self._refresh()