大数据分析与可视化 之 猫眼电影爬虫
import random
import time
import re
import requests
import csv
class MaoyanSpider(object):
# 初始化
# 定义初始页面url
def __init__(self):
self.url = 'https://www.maoyan.com/board/4?offset={}'
def get_html(self,url):
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept - Encoding':'gzip, deflate, br',
'Accept - Language':'zh - CN, zh;q = 0.9, en;q = 0.8, en - GB;q = 0.7, en - US;q = 0.6',
'Cache - Control':'max - age = 0',
'Connection':'keep-alive',
'Cookie':'__mta=142521997.1695026370028.1695105604302.1695106030738.15; uuid_n_v=v1; uuid=E13BF08055FE11EEBD786D8B351D1BEB69DF40D3DDA545AF98A7B7303437B12C; _lxsdk_cuid=18aa771d411c8-0d1e3cae321997-78505774-16e360-18aa771d411c8; _lxsdk=E13BF08055FE11EEBD786D8B351D1BEB69DF40D3DDA545AF98A7B7303437B12C; _csrf=db721765308a1674e7cba19e5867dbf60fb59d96717ba3913e47e499ab4726c0; Hm_lvt_703e94591e87be68cc8da0da7cbd0be2=1695026370,1695086835; _lx_utm=utm_source%3Dbing%26utm_medium%3Dorganic; __mta=142521997.1695026370028.1695087067256.1695090012297.10; Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2=1695106030; _lxsdk_s=18aac26180a-d75-779-1ac%7C%7C9',
'Host':'www.maoyan.com',
'Referer':'https://www.maoyan.com/board?requestCode=1d65f67ff588bed531de86b8d57274ac6pskg',
'Sec-Ch-Ua':'"Microsoft Edge"; v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
'Sec-Ch-Ua-Mobile':'?0',
'Sec-Ch-Ua-Platform':'"Windows"',
'Sec-Fetch-Dest':'document',
'Sec-Fetch-Mode':'navigate',
'Sec-Fetch-Site':'same-origin',
'Sec-Fetch-User':'?1',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31'
}
res = requests.get(url=url, headers=headers)
html = res.text
#测试
# print(res.text)
# 直接调用解析函数
self.parse_html(html)
def parse_html(self,html):
# 正则表达式
re_bds = '<div class="movie-item-info">.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?class="releasetime">(.*?)</p>'
# 生成正则表达式对象
pattern = re.compile(re_bds, re.S)
r_list = pattern.findall(html)
# 测试
# print(r_list)
self.save_html(r_list)
def save_html(self,r_list):
# 生成文件对象
with open('maoyan.csv', 'a', newline='', encoding="utf-8") as f:
# 生成csv操作对象
writer = csv.writer(f)
# 整理数据
for r in r_list:
name = r[0].strip()
star = r[1].strip()[3:]
# 上映时间：2018-07-05
# 切片截取时间
time = r[2].strip()[5:15]
L = [name, star, time]
# 写入csv文件
writer.writerow(L)
print(name, time, star)
# 主函数
def run(self):
# 抓取前6页数据
for offset in range(0, 61, 10):
url = self.url.format(offset)
self.get_html(url)
# 生成1-2之间的浮点数
time.sleep(random.uniform(1, 2))
# 以脚本方式启动
if __name__ == '__main__':
# 捕捉异常错误
try:
spider = MaoyanSpider()
spider.run()
except Exception as e:
print("错误:", e)