Scrapy中搭配Splash丶selenium

 1 # Splash类似Selenium模拟浏览器浏览网页来动态爬取网站
 2 # 文档:https://splash.readthedocs.io/en/stable/
 3 # Splash 在docker中安装: https://splash.readthedocs.io/en/stable/install.html#linux-docker  (docker笔记点这里)
 4 # docker pull scrapinghub/splash
 5 # 启动Splash
 6 # docker run -it -p 8050:8050 --rm scrapinghub/splash
 7 # https://splash.readthedocs.io/en/stable/faq.html?highlight=scrapy#python-scrapy
 8 # pip3 install scrapy-splash
 9 
10 import requests
11 from fake_useragent import UserAgent
12 from urllib.parse import quote
13 
14 headers = {
15     'User-Agent': UserAgent().chrome
16 }
17 
18 # 调用splash,来控制(lua语言)抓取网页内容   (lua笔记点这里)
19 def main():
20     url = r"https://www.baidu.com"
21     lua_script = '''function main(splash, args)
22     splash:go('{url}')
23     splash:wait(1)
24     return splash:html()
25 end'''.format(url=url)
26     response = requests.get(r"http://localhost:8050/execute?lua_source={}".format(quote(lua_script)), headers=headers)
27     response.encoding = 'utf-8'
28     print(response.text)
29 
30 # splash直接访问网站
31 def test1():
32     url = r"https://www.baidu.com"
33     response = requests.get(r"http://localhost:8050/render.html?url={}".format(quote(url)), headers=headers)
34     response.encoding = 'utf-8'
35     print(response.text)
36 
37 if __name__ == '__main__':
38     test1()

 1 # Splash 配置
 2 # 配置Splash URL地址
 3 SPLASH_URL = 'http://localhost:8050/'
 4 # 下载器中间件
 5 DOWNLOADER_MIDDLEWARES = {
 6    "scrapy_splash.SplashCookiesMiddleware": 200,
 7    "scrapy_splash.SplashMiddleware": 201,
 8    "myproject.middlewares.MyprojectDownloaderMiddleware": 202
 9 }
10 # 去重过滤器
11 DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
12 # 使用Splash的Http缓存
13 HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

 1 import scrapy
 2 from scrapy_splash import SplashRequest
 3 from urllib.parse import quote
 4 
 5 
 6 class CnblogsSpider(scrapy.Spider):
 7     name = "cnblogs"
 8     allowed_domains = ["cnblogs.com"]
 9     # start_urls = ["https://www.cnblogs.com/watermeloncode/"]
10 
11     def parse(self, response):
12         print(response.text)
13 
14     def parse1(self, response):
15         print(response.text)
16 
17     def start_requests(self):
18         lua_script = '''
19         function main(splash, args)
20             splash:go('{url}')
21             splash:wait(1)
22             return splash:html()
23         end'''.format(url=r"https://www.cnblogs.com/watermeloncode/")
24         # 要是执行lua代码的话就需要指定endpoint='execute', args={'lua_source':lua_script}
25         # yield SplashRequest(r"https://www.cnblogs.com/watermeloncode/", callback=self.parse, endpoint='execute', args={'lua_source':lua_script, 'wait':1})
26         yield SplashRequest(r"https://www.cnblogs.com/watermeloncode/", callback=self.parse, endpoint='render.html', args={'wait': 1})

 1 1.配置下载器
 2 class MyprojectDownloaderMiddleware:
 3     def process_request(self, request, spider):
 4         if spider.name == 'guazi':
 5             url = request.url
 6             # 在这里调用浏览器get请求
 7             spider.chrome.get(url)
 8             html = spider.chrome.page_source
 9             # 在这里返回HtmlResponse对象,后面就不会再次调用requests去发起请求了.因为这里我们已经把请求到的页面放置进去了.
10             return HtmlResponse(url=url, body=html, request=request, encoding='utf-8')
11             
12 2.Spider中控制创建浏览器与关闭浏览器
13 import scrapy
14 from scrapy import signals
15 from selenium import webdriver
16 
17 class GuaziSpider(scrapy.Spider):
18     name = "guazi"
19     allowed_domains = ["guazi.com"]
20     start_urls = ["https://www.guaz.com/"]
21 
22     @classmethod
23     def from_crawler(cls, crawler, *args, **kwargs):
24         # Spider对象的创建都在这里
25         # 记得调用父类方法创建好Spider对象
26         spider = super(GuaziSpider, cls).from_crawler(crawler, *args, **kwargs)
27         # 添加一个关闭时需执行的方法.这里在处理spider_closed信号的时候会执行spider.spider_closed回调
28         crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
29         # 给Spider对象中内置一个浏览器对象
30         spider.chrome = webdriver.Chrome()
31         return spider
32 
33     def spider_closed(self, spider):
34         # 收到signals.spider_closed信号的时候执行该方法,关闭浏览器
35         spider.chrome.quit()
36         print('close selenium')
37 
38     def parse(self, response):
39         print(response.text)