Python实践(1):使用爬虫将小说保存为txt文件

# !/usr/bin/python
# -*-coding:utf-8-*-

import requests
from bs4 import BeautifulSoup

def getContent(path):
    Encoding='gbk' # 设置相应的字符集,如果出现乱码,可尝试更换为utf-8
    if(path[0:7]=="http://"):
        #从网络获取
        response = requests.get(path)
        response.encoding = Encoding
        return response.text
    else:
        #从本地获取
        return open(path,encoding=Encoding).read()

    

# dist="http://book.sbkk8.com/waiguo/anna_kalienina/"
dist="./《安娜·卡列尼娜》小说在线阅读_列夫·托尔斯泰.html"

data = getContent(dist)
soup = BeautifulSoup(data,"html.parser")
f=open("urls.txt","w",buffering=1)

for i in range(1,68):
    chapters=soup.select("#left > div.mulu > ul > li:nth-child("+str(i)+") > a")
    print(chapters[0].get('href'))
    f.write(chapters[0].get('href')+'\n')

# !/usr/bin/python
# -*-coding:utf-8-*-

import requests
from bs4 import BeautifulSoup
import time

def getContent(path):
    Encoding='gbk' # 设置相应的字符集,如果出现乱码,可尝试更换为utf-8
    if(path[0:4]=="http"):
        #从网络获取
        response = requests.get(path)
        response.encoding = Encoding
        return response.text
    else:
        #从本地获取
        return open(path,encoding=Encoding).read()


f=open("urls.txt","r",buffering=1)
urls=map(lambda x:x.replace(' ','').replace('\r','').replace('\n',''),f.readlines())
f2=open("安娜·卡列尼娜_列夫·托尔斯泰.txt","w",buffering=1)

for u in urls:
    time.sleep(1)
    data=getContent(u)
    soup = BeautifulSoup(data,"html.parser")
    title=soup.title.string
    content=soup.select("#content")[0].get_text()
    print(title)
    print(content)
    f2.write(title+'\n')
    f2.write(content+'\n')