python爬虫--小说爬取实战

代码

import urllib.request
import re
import time
headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
urllib.request.install_opener(opener)#以上是伪装成网页

url="http://www.doupobook.com/doupo"
data=urllib.request.urlopen(url).read().decode('utf-8','ignore')
pat='<li><a href=".*?title="(.*?)"'#爬取章节名称的正则表达式
pat_1='<li><a href="(.*?)"'#爬取章节的网址的正则表达式
pat_2='<div class="m-post"><p>(.*?)</div>'#文章的爬取的正则表达式
result=re.compile(pat).findall(data)
result_1=re.compile(pat_1).findall(data)
q=0
for i in result_1:
    data_1=urllib.request.urlopen(i).read().decode('utf-8','ignore')
    result_2=re.compile(pat_2).findall(data_1)
    file="C:/Users/F.S.Z/Desktop/新建文件夹"+"/"+result[q]+".doc"
    with open(file,'w') as f:
        result_2[0]=result_2[0].replace('</p>','\r\n')
        result_2[0] = result_2[0].replace('<p>', ' ')
        f.write(result_2[0])
    print(result[q]+" success!!!")
    time.sleep(3)
    q+=1

运行结果