Python学习之爬虫07-糗事百科段子爬取

Python学习之爬虫07-糗事百科段子爬取

概述：

巩固练习。

#糗事百科段子爬虫
import urllib.request
import re
headers=("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
#安装为全局
urllib.request.install_opener(opener)
for i in range(0,35):
    thisurl="http://www.qiushibaike.com/8hr/page/"+str(i+1)+"/?s=4948859"
    data=urllib.request.urlopen(thisurl).read().decode("utf-8","ignore")
    pat='<div class="content">.*?<span>(.*?)</span>.*?</div>'
    rst=re.compile(pat,re.S).fiimport urllib.request
import re 
# 添加报头并将报头安装为全局属性
headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3493.3 Safari/537.36")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
urllib.request.install_opener(opener)
pat='<div class="content">.*?<span>(.*?)</span>.*?</div>'
fh=open("G:\\Python_Test\\qiushibaike\\qiushibaike.txt","a+",encoding="utf-8")
for i in range(1,10):
    url="https://www.qiushibaike.com/text/page/"+str(i)+"/"
    data=urllib.request.urlopen(url).read().decode("utf-8","ignore")
    #print(data)
    rst=re.compile(pat,re.S).findall(data)  # re.S为模式修正符，使得 .可以匹配换行符,这样就可以匹配多行数据
    for j in range(0,len(rst)):
        fh.write(rst[j])
        #fh.write("\n")
        #print(rst[j])
        #print("----------------")
    print("当前(第"+str(i)+"页）爬取成功！")
fh.close()ndall(data)
    for j in range(0,len(rst)):
        print(rst[j])
        print("-------")