python爬取京东商品(使用cookies,header,xpath匹配),解决ajax。 后附有selenium模拟访问抓取网页方法和视频演示
###本文提供的代码只用于学习,请勿用于商业用途
这次没用正则和BS4,用了XPATH,因为真的好久好久没用xpath了。
唯一要主要克服的难点就是京东的ajax加载,直接requests抓网页只有前30个商品,还要用get请求访问目标url,得到后面的商品。b变量其实是UNIX时间戳。
PS,真的好想买索尼的电视机,因为安卓电视+无开机广告......建议电视机运行内存大于2G,CPU配A73,选择安卓是为后续刷机或者装第三方应用考虑
#coded by 伊玛目的门徒
from bs4 import BeautifulSoup
import requests
# coding: utf-8
from lxml import etree
import pandas as pd
import time
start = time.clock()
header={'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.XXXX.XXX Safari/537.36'}
head={'authority': 'search.jd.com',
'method': 'GET',
'path': '/s_new.php?keyword=%E7%94%B5%E8%A7%86%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E7%94%B5%E8%A7%86%E6%9C%BA',
'scheme':'https',
'referer': 'https://search.jd.com/Search?keyword=%E7%94%B5%E8%A7%86%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E7%94%B5%E8%A7%86%E6%9C%BA&cid2=653&cid3=655&page=3&s=58&click=0',
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
'x-requested-with': 'XMLHttpRequest',
'Cookie':'pinId=P7U4oU1101vnxG4Eir9OvbV9-x-f3wj7; shshshfpa=3120508e-1db3-33c7-d34d-733fc57f5c61-1527256832; shshshfpb=2b679cd3d88844dc7956f01c306d1e8ea5b0817076f66fb12543702016; qrsc=3; __jdu=2045744036; TrackID=1wgah744lMsZWXTwC-q-Biyu37x7sxp8CseoInLtJZAVBG6zQJux5G9mvMGBhV3T3hMRrtlFHyNDJTPVf9ckESDMKHNDoZr-Wr6ETbyEoNGE; __jdv=122270672|baidu|-|organic|not set|1552825203464; areaId=2; PCSYCityID=2; shshshfp=f1024c8fe38dc72f7cbd01d3e8120c95; xtest=428.cf6b6759; ipLoc-djd=2-2830-51800-0; user-key=859e46c8-d778-4972-8cb6-435b9d3378b6; cn=0; 3AB9D23F7A4B3C9B=J6H7CPQOXRM4BEEC34ESMFZSPDECXYZUVXQW2BOOW2P3QJXJOXSEAMVSMD5DV3NK7MR3DZNGSK2XAS6OUUNRDR532U; __jda=122270672.2045744036.1495102986.1552825203.1552906130.51; __jdc=122270672; rkv=V0500; __jdb=122270672.2.2045744036|51.1552906130; shshshsID=8782cfd99609426bed86f079aa971f79_2_1552906303039'
}
cookies={'cookies': 'pinId=P7U4oU1101vnxG4Eir9OvbV9-x-f3wj7; shshshfpa=3120508e-1db3-33c7-d34d-733fc57f5c61-1527256832; shshshfpb=2b679cd3d88844dc7956f01c306d1e8ea5b0817076f66fb12543702016; qrsc=3; __jdu=2045744036; TrackID=1wgah744lMsZWXTwC-q-Biyu37x7sxp8CseoInLtJZAVBG6zQJux5G9mvMGBhV3T3hMRrtlFHyNDJTPVf9ckESDMKHNDoZr-Wr6ETbyEoNGE; __jda=122270672.2045744036.1495102986.1541941006.1552825203.50; __jdc=122270672; __jdv=122270672|baidu|-|organic|not set|1552825203464; areaId=2; PCSYCityID=2; shshshfp=f1024c8fe38dc72f7cbd01d3e8120c95; xtest=428.cf6b6759; rkv=V0500; 3AB9D23F7A4B3C9B=J6H7CPQOXRM4BEEC34ESMFZSPDECXYZUVXQW2BOOW2P3QJXJOXSEAMVSMD5DV3NK7MR3DZNGSK2XAS6OUUNRDR532U; ipLoc-djd=2-2830-51800-0; __jdb=122270672.4.2045744036|50.1552825203; shshshsID=99df96dc146b5071a6934e385d2d7f33_4_1552826030047'}
name=[]
p=[]
def frontpage(n):
url='https://search.jd.com/Search?keyword=电视机&enc=utf-8&qrst=1&rt=1&stop=1&page='+str(2*n-1)
#html=requests.get(url,headers=header,cookies=cookies)
html=requests.get(url,headers=head)
html.encoding = 'utf-8'
selector = etree.HTML(html.text)
goods_name = selector.xpath('//div/a/em/text()[1]')
for i in goods_name:
print (i)
name.append(i)
price = selector.xpath('//div/strong/i')
for i in price:
print (i.text)
p.append(i.text)
print (len(goods_name))
print (len(price))
x1=selector.xpath('//li[19]/div/div[2]/strong/i')
print ('###############################################')
def backpage(n):
a=time.time()
b='%.5f'%a
url_last='https://search.jd.com/s_new.php?keyword=%E7%94%B5%E8%A7%86%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&page='+str(2*n)+'&s='+str(48*n-20)+'&scrolling=y&log_id='+str(b)
url_test='https://search.jd.com/s_new.php?keyword=%E7%94%B5%E8%A7%86%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&page='+str(2*n)+'&s='+str(48*n-20)+'&scrolling=y&log_id='+str(b)
html2=requests.get(url_test,headers=head)
html2.encoding = 'utf-8'
selector = etree.HTML(html2.text)
goods_name = selector.xpath('//div/a/em/text()[1]')
for i in goods_name:
print (i)
name.append(i)
print (len(goods_name))
price = selector.xpath('//div/strong/i')
for i in price:
print (i.text)
p.append(i.text)
print (len(price))
print ('###############################################')
for n in range(1,10):
try:
frontpage(n)
except Exception as e:
print(e)
print ('fail')
print (2*n-1)
try:
backpage(n)
except Exception as e:
print(e)
print ('fail')
print (2*n)
df=pd.DataFrame({
'goods_name':name,
'price':p
})
print (df)
end = time.clock()
print ('用时: %f '%(end-start))
print('################保存csv####################')
df.to_csv('京东电视机.csv')
爬虫结果:
视频演示:
https://www.bilibili.com/video/av46769501/
还有一个我借鉴别人的chrome模拟访问方式(selenium)以实现ajax加载,但有一说一,真的很慢,建议多线程
# -*- coding: utf-8 -*-
from selenium import webdriver
from bs4 import BeautifulSoup
import time
print ('yes')
driver = webdriver.Chrome()
#s1='%E8%A3%A4%E5%AD%90'
s1='%E7%94%B5%E8%A7%86'
url='https://search.jd.com/Search?keyword='+s1+'&enc=utf-8&page=1'
driver.get(url)
print (driver.current_url)
t1 = time.time()
n = 10 #这里可以控制网页滚动距离
for i in range(1,n+1):
s = "window.scrollTo(0,document.body.scrollHeight/{0}*{1});".format(5,i)
print (s, len(driver.page_source),time.time()-t1)
driver.execute_script(s)
time.sleep(0.5)
lis=driver.find_elements_by_class_name("gl-item")
for i in range(len(lis)):
print (lis[i].find_element_by_class_name("p-name").text)
print (lis[i].find_element_by_class_name("p-price").text)
# print lis[i].text
driver.quit()
print ("end")