豆瓣图书排行250抓取练习
第二天,利用BeautifulSoup解析库
import requests
import re
import time
from bs4 import BeautifulSoup
def get_one_page(url):
proxies = {"http":"http://61.145.69.27","http":"http://121.61.0.208"}
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}
response = requests.get(url,headers=headers,proxies=proxies)
if response.status_code == 200:
return response.text
return None
def parse_one_page(html):
soup = BeautifulSoup(html, 'lxml')
for td in soup.find_all(attrs={'valign':'top'}):
for a in td.find(name='a'):
print(a.string)
for p in td.find_all(name='p'):
print(p.string)
for span in td.find_all(name='span'):
print(span.string)
def main(offset):
url = 'https://book.douban.com/top250?start='+str(offset)
html = get_one_page(url)
parse_one_page(html)
if __name__ == '__main__':
for i in range(10):
main(i * 25)
time.sleep(1)
运行结果:
不知道如何选择抓取节点,结果中有许多[None]