大数据分析与可视化 之 百度贴吧爬虫
import csv
import datetime
import json
from urllib import request, parse
import time
import random
from fake_useragent import UserAgent
from lxml import etree
import requests
# 定义一个爬虫类
class TiebaSpider(object):
# 初始化url属性
def __init__(self):
self.url = 'http://tieba.baidu.com/f?{}'
# 1.请求函数，得到页面，传统三步
def get_html(self, url):
ua = UserAgent()
response = requests.get(url=url, headers={'User-Agent': ua.random})
# res = request.urlopen(req)
# windows会存在乱码问题，需要使用 gbk解码，并使用ignore忽略不能处理的字节
# linux不会存在上述问题，可以直接使用decode('utf-8')解码
html = response.content.decode('utf-8').replace("<!--", "").replace("-->", "")
return html
# 2.解析函数，此处代码暂时省略，还没介绍解析模块
def parse_html(self, html):
eroot = etree.HTML(html)
# 提取行数据
li_list = eroot.xpath('//*[@id="thread_list"]/li/div/div[2]/div[1]/div[1]/a')
data = []
for li in li_list:
item = {}
item["title"] = li.xpath('./text()')[0]
item["link"] = 'https://tieba.baidu.com' + li.xpath('./@href')[0]
data.append(item)
return data
def save_csv(self, data, filename):
with open(filename, 'a', newline='', encoding='utf_8_sig') as csv_file:
# 指定CSV文件的列名
fieldnames = ['title', 'link']
# 创建CSV写入器
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
# 写入列名
writer.writeheader()
# 写入数据
for row in data:
writer.writerow(row)
print(f'Data has been written to {filename}')
# def save_items(self, items, filename):
# for data in items:
# # 将字典转换为字符串，将字符串编码为UTF-8字节流
# data_str = json.dumps(data).encode("utf-8")
# data_str = data_str.decode("unicode_escape")
# self.write(data_str, filename)
#
# def write(self, item, filename):
# # 打开文件进行追加写入
# with open(filename, "a", encoding="utf-8") as file:
# # 写入数据 将字节流解码为字符串
# file.write(item)
# file.write("\n")
# 3.保存文件函数
# def save_html(self, filename, html):
# with open(filename, 'w') as f:
# f.write(html)
# 4.入口函数
def run(self):
# current_datetime = datetime.datetime.now()
# 构建文件名，精确到小时和分钟
# filename = current_datetime.strftime("%Y-%m-%d-%H-%M.csv")
name = input('输入贴吧名：')
begin = int(input('输入起始页：'))
stop = int(input('输入终止页：'))
# +1 操作保证能够取到整数
for page in range(begin, stop + 1):
pn = (page - 1) * 50
params = {
'kw': name,
'pn': str(pn)
}
# 拼接URL地址
params = parse.urlencode(params)
url = self.url.format(params)
# 发请求
html = self.get_html(url)
items = self.parse_html(html)
# self.save_items(items, filename)
filename = '{}-{}页.csv'.format(name, page)
self.save_csv(items, filename)
# 定义路径
# filename = '{}-{}页.html'.format(name, page)
# self.save_html(filename, html)
# 提示
print('第%d页抓取成功' % page)
# 每爬取一个页面随机休眠1-2秒钟的时间
time.sleep(random.randint(1, 2))
# 以脚本的形式启动爬虫
if __name__ == '__main__':
start = time.time()
spider = TiebaSpider() # 实例化一个对象spider
spider.run() # 调用入口函数
end = time.time()
# 查看程序执行时间
print('执行时间:%.2f' % (end - start)) # 爬虫执行时间