爬取诗词网站中的文章

 1 import requests
 2 # 实例化BeautifulSoup对象
 3 from bs4 import BeautifulSoup
 4 if __name__ == "__main__":
 5     #设置User-Agent将爬虫伪装成用户通过浏览器访问
 6     header = {
 7         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36 Edg/87.0.664.55'
 8     }
 9     #要访问的网页url地址
10     url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
11     #发起请求
12     respond = requests.get(url=url,headers=header)
13     #通过实例化获取网页源码
14     soup1 = BeautifulSoup(respond.content,'lxml')
15     #select返回列表，找到连接标签
16     title = soup1.select('.book-mulu a')
17     #打开sanguo.txt文件，设置字节码格式避免乱码
18     fp = open('./sanguo.txt','w',encoding='utf-8')
19     #循环title列表里的链接
20     for i in title:
21         #通过.string获取a链接下的直系文本作为标题
22         title = i.string
23         #补全a连接，获取特定的href属性
24         data_url = "https://www.shicimingju.com"+i['href']
25         #对a连接的url进行请求，进一步获取链接里的文章
26         soup2 = BeautifulSoup(requests.get(url=data_url,headers=header).content,'lxml')
27         #找到文章所在的标签
28         content = soup2.find('div',class_='chapter_content')
29         #将文章标题及其文章的内容获取到写入刚刚打开的文件
30         fp.write(title+":"+content.text+"\n")
31         print(title+"爬取成功")
32     print("Over")