随机切换UA中间件实例

随机UA下载中间件(初始版)

setting文件:

1
2
3
DOWNLOADER_MIDDLEWARES = {
'ArticleSpider.middlewares.RandomUserAgentMiddleware': 543,
}

middlewares文件:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
from fake_useragent import UserAgent
class RandomUserAgentMiddleware(object):
def __init__(self, crawler):
super(RandomUserAgentMiddleware, self).__init__()
self.ua = UserAgent()
self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random')

@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request, spider):
def get_ua():
return getattr(self.ua, self.ua_type)
request.headers.setdefault('User-Agent', get_ua())

随机UA、ip下载中间件(升级版)

setting文件:

1
2
3
4
DOWNLOADER_MIDDLEWARES = {
# 'ArticleSpider.middlewares.MyCustomDownloaderMiddleware': 543,
'ArticleSpider.middlewares.rotate_user_agent.RandomUserAgentMiddleware': 400,
}

middlewares文件:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
from fake_useragent import UserAgent
class RandomUserAgentMiddleware(object):
def __init__(self, crawler):
super(RandomUserAgentMiddleware, self).__init__()

self.ua = UserAgent()
self.per_proxy = crawler.settings.get('RANDOM_UA_PER_PROXY', False)
self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random')
self.proxy2ua = {}

@classmethod
def from_crawler(cls, crawler):
return cls(crawler)

def process_request(self, request, spider):
def get_ua():
'''Gets random UA based on the type setting (random, firefox…)'''
return getattr(self.ua, self.ua_type)

if self.per_proxy:
proxy = request.meta.get('proxy')
if proxy not in self.proxy2ua:
self.proxy2ua[proxy] = get_ua()
logger.debug('Assign User-Agent %s to Proxy %s'
% (self.proxy2ua[proxy], proxy))
request.headers.setdefault('User-Agent', self.proxy2ua[proxy])
else:
ua = get_ua()
request.headers.setdefault('User-Agent', get_ua())

煌金 wechat
扫描关注公众号,回复「1024」获取为你准备的特别推送~
  • 本文作者: 煌金 | 微信公众号【咸鱼学Python】
  • 本文链接: http://www.xianyucoder.cn/2018/11/29/ua-downloadmiddler/
  • 版权声明: 本博客所有文章除特别声明外,均采用 许可协议。转载请注明出处!
  • 并保留本声明和上方二维码。感谢您的阅读和支持!