爬虫作业-526互联

（一）
import requests
from bs4 import BeautifulSoup


def getUrlText(url):
    try:
        web = requests.get(url)
        soup = BeautifulSoup(web.text)
        web.raise_for_status()             
        web.encoding = 'utf-8'          
        return web.text, web.status_code, len(web.text), web.encoding, len(soup.text)

    except:
        return


url = "https://www.sogou.com"

for i in range(20):
    print(i)
    print(getUrlText(url))



（二）

 from bs4 import BeautifulSoup
 import re
 path = 'C:/Users/huanghy/Desktop/code.html'
 htmlfile = open(path, 'r', encoding='utf-8')
 htmlhandle = htmlfile.read()
 soup=BeautifulSoup(htmlhandle, "html.parser")
 print(soup.head,"33")
 print(soup.body)
 print(soup.find_all(id="first")) r=soup.text
 pattern = re.findall('[\u4e00-\u9fa5]+',r)
 print(pattern)



（三）

import bs4
import requests
from bs4 import BeautifulSoup
import pandas as pd
import matplotlib.pyplot as plt

def getHTMLText(url):
    try:
        res = requests.get(url,timeout = 30)
        res.raise_for_status()
        res.encoding = res.apparent_encoding
        return res.text
    except:
        return "访问未成功"


def fillUnivList(ulist, html):  # 将一个html页面放入一个列表
    soup = BeautifulSoup(html, "html.parser")
    # 每个<tr>包含一所大学的所有信息
    # 所有<tr>信息包在<tbody>中
    for tr in soup.find('tbody').children:
        if isinstance(tr, bs4.element.Tag):  # 过滤掉非标签信息，以取出包含在<tr>标签中的bs4类型的Tag标签
            tds = tr('td')  # 等价于tr.find_all('td'),在tr标签中找td标签内容
            # print(tds)
            ulist.append([tds[0].string, tds[1].string, tds[3].string, tds[2].string])
            # td[0],[1],[3],[2]，分别对应每组td信息中的排名，学校名称，得分，区域。将这些信息从摘取出来
            print(ulist)
    return ulist

def writedata(ulist,file):
    where_list = []
    dict = {}
    df = pd.DataFrame(ulist,columns=['排名','学校名称','得分','区域'])  #list转dataframe
    df.to_csv(file,',',index=False,encoding="gbk")
    print("写入完成！")
    for i in range(100):
        if df.iloc[i,-1] in where_list:
            dict[df.iloc[i,-1]] += 1
        else:
            where_list.append(df.iloc[i,-1])
            dict[df.iloc[i,-1]] = 1
    print(dict)

    return dict


if __name__ == '__main__':
    uinfo = []
    url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
    soup = getHTMLText(url)
    ulist = fillUnivList(uinfo,soup)
    file = "D:\\tt.csv"
    dict = writedata(ulist,file)

爬虫