526互联

Python爬虫(二):写一个爬取壁纸网站图片的爬虫(图片下载,词频统计,思路)

发布时间 2023-06-19 18:10:40作者: 养肥胖虎

好家伙，写爬虫

 

代码:

 

import requests
import re
import os
from collections import Counter
import xlwt
# 创建Excel文件
workbook = xlwt.Workbook(encoding='utf-8')
worksheet = workbook.add_sheet('url_list')

# 将数据写入Excel文件
worksheet.write(0, 0, '序号') #写入对应的字段
worksheet.write(0, 1, '图片详细地址')
worksheet.write(0, 2, '图片TAG')

def get_response(html_url):
    headers ={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43'
    }
    response = requests.get(url=html_url,headers=headers)
    return response

def get_img_src(html_url):
    #获取榜单地址
    response =get_response(html_url)
    list_url =re.findall('<img src="(.*?)"',response.text)
    return list_url

def get_img_src_ciping(html_url):
    #获取榜单地址
    response =get_response(html_url)
    # print(response.text)
    list_url =re.findall('title="(.*?)">',response.text)
    if(list_url):
        return list_url[0]
    else:
        return "无内容"
        
url = 'https://wallspic.com/cn/topic/pc_wallpapers'
response = get_response(html_url=url)
html_code = response.text
# print(html_code)

# 定义正则表达式匹配模式
pattern = r'"link":"(.*?)"'

# 使用re.findall()方法获取所有匹配结果
result_list = re.findall(pattern, html_code)

cleaned_urls_1 = []
for url in result_list:
    cleaned_url = url.replace("\\/\\/", "/")
    cleaned_url = url.replace("\\/", "/")
    cleaned_urls_1.append(cleaned_url)
    print(cleaned_url)


cleaned_urls_2 = []
cleaned_urls_3 = []
for url in cleaned_urls_1:
    # 使用os.path.splitext()方法将URL路径拆分为文件名和扩展名
    filename, ext = os.path.splitext(url)
    # 判断扩展名是否为.jpg
    if ext.lower() == '.jpg':
        cleaned_urls_2.append(url)
    if ext.lower() != '.jpg' and ext.lower() !='.webp':
        cleaned_urls_3.append(url)

# print(cleaned_urls_2)

save_dir = 'C:/Users/10722/Desktop/python答辩/canuse/img/'  # 指定保存路径

if not os.path.exists(save_dir):
    os.makedirs(save_dir)

#下载图片
row = 1
for url in cleaned_urls_2:
    worksheet.write(row, 0, row+1) #将排行写入excel表格
    worksheet.write(row, 1, url)          #将歌名写入excel表格

    filename = os.path.basename(url)  # 获取文件名
    filepath = os.path.join(save_dir, filename)  # 拼接保存路径和文件名
    
    response = requests.get(url)
    with open(filepath, 'wb') as f:
        f.write(response.content)
        print(f'{filename} 下载完成')
    row+=1

print('全部图片下载完成')

cleaned_urls_4 =[]
roww = 1
for url in cleaned_urls_3:
    # print(url)
    
    response=get_img_src_ciping(html_url=url)
    # print(response)

    worksheet.write(roww, 2, response)        #将tag写入excel表格
    roww+=1
    cleaned_urls_4.append(response)
    

print(cleaned_urls_4)
# urls=str(cleaned_urls_4)
# 将数组中的字符串拼接成一个长字符串
long_string = " ".join(cleaned_urls_4)

# 使用空格将长字符串分割成一个单词列表
word_list = long_string.split()

# 使用Counter对单词列表进行词频统计
word_counts = Counter(word_list)

words =str(word_counts)

worksheet.write(roww, 2, words)        #将歌手写入excel表格
print(word_counts)
workbook.save('C:/Users/10722/Desktop/python答辩/canuse/图片详情.xls')
Pachong.py

 

效果展示

 

 

上次爬虫去答辩了，爬的酷狗音乐的某个排行榜，

老师问我为什么没有把音乐也爬下来，

然后去翻了一下，这个音乐加密有点超出我的能力范围了(他真的太会藏了)

 

重写吧

我决定换个网站,爬个图片吧

找个壁纸网站

最佳 电脑壁纸 [30000+] |在 Wallspic 上免费下载

 

看上去好爬

 可以直接拿到图片的src

开干！

 

依旧按照以下思路进行

1.发请求，随后拿到服务器发过来的.html文件

2.用正则表达式去套对应的，我们需要的数据

3.处理数据，最后把他们以某种方式呈现

 

1.发请求拿到.html文件 

url = 'https://wallspic.com/cn/topic/pc_wallpapers'
response = get_response(html_url=url)
html_code = response.text
print(html_code)

来看看拿到的.html文本

我们并不能很直观的直接拿到<img>标签中的src

他似乎是通过某种方式注入

 

2.拿到图片的src

但是没有太大问题,我们依旧能够找到其中的src

用一个方法()把他们"过滤出来"

cleaned_urls_1 = []
for url in result_list:
    cleaned_url = url.replace("\\/\\/", "/")
    cleaned_url = url.replace("\\/", "/")
    cleaned_urls_1.append(cleaned_url)
    print(cleaned_url)

 来看看

 这里可以看到,在这些地址中,以两条为一组,其中,第一条为该壁纸的详情页,第二条为该壁纸的真正src

随后,可以想个办法把他们分别放到不同的数组中

cleaned_urls_2 = []
cleaned_urls_3 = []
for url in cleaned_urls_1:
    # 使用os.path.splitext()方法将URL路径拆分为文件名和扩展名
    filename, ext = os.path.splitext(url)
    # 判断扩展名是否为.jpg
    if ext.lower() == '.jpg':
        cleaned_urls_2.append(url)
    if ext.lower() != '.jpg' and ext.lower() !='.webp':
        cleaned_urls_3.append(url)

这里偷个懒,直接使用第三方插件os去做一个拆分,然后用ext.lower拿到他们的后缀,再进行一个判断

把两种地址分类,把他们放到不同的数组中,

详情页在后面有用,要对图片的TAG进行一个词频统计

 

3.下载图片到本地

save_dir = 'C:/Users/10722/Desktop/python答辩/canuse/img/'  # 指定保存路径

if not os.path.exists(save_dir):
    os.makedirs(save_dir)

#下载图片
row = 1
for url in cleaned_urls_2:
    worksheet.write(row, 0, row+1) #将排行写入excel表格
    worksheet.write(row, 1, url)          #将歌名写入excel表格

    filename = os.path.basename(url)  # 获取文件名
    filepath = os.path.join(save_dir, filename)  # 拼接保存路径和文件名
    
    response = requests.get(url)
    # with open(filepath, 'wb') as f:
    #     f.write(response.content)
        # print(f'{filename} 下载完成')
    row+=1

print('全部图片下载完成')

这里用response发一个get请求就可以了,

 

 

4.词频统计

这里同样的,我们去到图片的详情页,然后用一个正则表达式去套图片的TAG

def get_img_src_ciping(html_url):
    #获取榜单地址
    response =get_response(html_url)
    # print(response.text)
    list_url =re.findall('title="(.*?)">',response.text)
    if(list_url):
        return list_url[0]
    else:
        return "无内容"

 

再使用插件collections去统计词频

cleaned_urls_4 =[]
roww = 1
for url in cleaned_urls_3:
    # print(url)
    
    response=get_img_src_ciping(html_url=url)
    # print(response)

    worksheet.write(roww, 2, response)        #将tag写入excel表格
    roww+=1
    cleaned_urls_4.append(response)
    
print(cleaned_urls_4)
# urls=str(cleaned_urls_4)
# 将数组中的字符串拼接成一个长字符串
long_string = " ".join(cleaned_urls_4)

# 使用空格将长字符串分割成一个单词列表
word_list = long_string.split()

# 使用Counter对单词列表进行词频统计
word_counts = Counter(word_list)

words =str(word_counts)

 

然后就,搞定啦

完整代码：

import requests
import re
import os
from collections import Counter
import xlwt
# 创建Excel文件
workbook = xlwt.Workbook(encoding='utf-8')
worksheet = workbook.add_sheet('url_list')

# 将数据写入Excel文件
worksheet.write(0, 0, '序号') #写入对应的字段
worksheet.write(0, 1, '图片详细地址')
worksheet.write(0, 2, '图片TAG')

def get_response(html_url):
    headers ={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43'
    }
    response = requests.get(url=html_url,headers=headers)
    return response

def get_img_src(html_url):
    #获取榜单地址
    response =get_response(html_url)
    list_url =re.findall('<img src="(.*?)"',response.text)
    return list_url

def get_img_src_ciping(html_url):
    #获取榜单地址
    response =get_response(html_url)
    # print(response.text)
    list_url =re.findall('title="(.*?)">',response.text)
    if(list_url):
        return list_url[0]
    else:
        return "无内容"
        


url = 'https://wallspic.com/cn/topic/pc_wallpapers'
response = get_response(html_url=url)
html_code = response.text
# print(html_code)

# 定义正则表达式匹配模式
pattern = r'"link":"(.*?)"'

# 使用re.findall()方法获取所有匹配结果
result_list = re.findall(pattern, html_code)

cleaned_urls_1 = []
for url in result_list:
    cleaned_url = url.replace("\\/\\/", "/")
    cleaned_url = url.replace("\\/", "/")
    cleaned_urls_1.append(cleaned_url)
    print(cleaned_url)


cleaned_urls_2 = []
cleaned_urls_3 = []
for url in cleaned_urls_1:
    # 使用os.path.splitext()方法将URL路径拆分为文件名和扩展名
    filename, ext = os.path.splitext(url)
    # 判断扩展名是否为.jpg
    if ext.lower() == '.jpg':
        cleaned_urls_2.append(url)
    if ext.lower() != '.jpg' and ext.lower() !='.webp':
        cleaned_urls_3.append(url)

# print(cleaned_urls_2)

save_dir = 'C:/Users/10722/Desktop/python答辩/canuse/img/'  # 指定保存路径

if not os.path.exists(save_dir):
    os.makedirs(save_dir)

#下载图片
row = 1
for url in cleaned_urls_2:
    worksheet.write(row, 0, row+1) #将排行写入excel表格
    worksheet.write(row, 1, url)          #将歌名写入excel表格

    filename = os.path.basename(url)  # 获取文件名
    filepath = os.path.join(save_dir, filename)  # 拼接保存路径和文件名
    
    response = requests.get(url)
    # with open(filepath, 'wb') as f:
    #     f.write(response.content)
        # print(f'{filename} 下载完成')
    row+=1

print('全部图片下载完成')

cleaned_urls_4 =[]
roww = 1
for url in cleaned_urls_3:
    # print(url)
    
    response=get_img_src_ciping(html_url=url)
    # print(response)

    worksheet.write(roww, 2, response)        #将tag写入excel表格
    roww+=1
    cleaned_urls_4.append(response)
    

print(cleaned_urls_4)
# urls=str(cleaned_urls_4)
# 将数组中的字符串拼接成一个长字符串
long_string = " ".join(cleaned_urls_4)

# 使用空格将长字符串分割成一个单词列表
word_list = long_string.split()

# 使用Counter对单词列表进行词频统计
word_counts = Counter(word_list)

words =str(word_counts)

worksheet.write(roww, 2, words)        #将歌手写入excel表格
print(word_counts)
workbook.save('C:/Users/10722/Desktop/python答辩/canuse/图片详情.xls')
# <img width="500" height="281" class="deferred_image deferred_image-clear" src="https://img2.wallspic.com/previews/2/2/8/3/7/173822/173822-kui_jia-dian_shi-chao_ji_ying_xiong-kong_jian-dong_zuo_mao_xian_de_you_xi-500x.jpg" alt="图片 盔甲, 电视, 超级英雄, 空间, 动作冒险的游戏">