利用python爬取mobike在西安地区的分布情况
先附上整个思路流程
1，获取mobike单车的API接口
2，爬取API数据
3，数据可视化
一：利用Fiddle抓取mobileAPI数据接口得到post请求如下
POST https://mwx.mobike.com/nearby/nearbyBikeInfo HTTP/1.1
charset: utf-8
Accept-Encoding: gzip
opensrc: list
eption: 0a235
time: 时间戳
subsource:
longitude:
mobileno:
content-type: application/x-www-form-urlencoded
referer: https://servicewechat.com/wx80f809371ae33eda/350/page-frame.html
mainsource: 4003
userid: 4084302776004076639232125813
platform: 4
wxapp: 1
lang: zh
latitude: 34.141276
citycode: 029
accesstoken: 941e812e4db1652fd753af5a81242ac4-0
locationtime: 1540817570095
User-Agent: Mozilla/5.0 (Linux; Android 7.1.2; Redmi 4X Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070336) NetType/WIFI Language/zh_CN Process/appbrand3
Content-Length: 184
Host: mwx.mobike.com
Connection: Keep-Alive
biketype=0&speed=0&errMsg=getLocation%3Aok&longitude=&verticalAccuracy=0&latitude=&accuracy=30&horizontalAccuracy=30&userid=4084302776004076639232125813&citycode=029
##二：分析链接
POST https://mwx.mobike.com/nearby/nearbyBikeInfo HTTP/1.1
charset: utf-8
Accept-Encoding: gzip
opensrc: list
eption: 0a235
time: 这里为当前时间戳
subsource:
longitude: x坐标以百度坐标系为准
mobileno:mobike账号
content-type: application/x-www-form-urlencoded
referer: https://servicewechat.com/wx80f809371ae33eda/350/page-frame.html
mainsource: 4003
userid: 4084302776004076639232125813
platform: 4
wxapp: 1
lang: zh
latitude: y坐标以百度坐标系为准
citycode: 029
accesstoken: 941e812e4db1652fd753af5a81242ac4-0
locationtime: 1540817570095
User-Agent: Mozilla/5.0 (Linux; Android 7.1.2; Redmi 4X Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070336) NetType/WIFI Language/zh_CN Process/appbrand3
Content-Length: 184
Host: mwx.mobike.com
Connection: Keep-Alive
biketype=0&speed=0&errMsg=getLocation%3Aok&longitude=&verticalAccuracy=0&latitude=&accuracy=30&horizontalAccuracy=30&userid=4084302776004076639232125813&citycode=029
##三：通过更改headers伪装成手机浏览器以获得返回数据
这里就不细说，简单的两个for循环
##四：数据可视化，利用百度地图提供的API进行热度分析
首先获得百度开发者的ak，然后就百度去吧。。。。。。
##代码（一）:
import requests
import time
import re
def mobike(x,y):
t = time.time()
num = int(t)
num = str(num)
num1=str(x)
num2=str(y)
url = 'https://mwx.mobike.com/nearby/nearbyBikeInfo'
doc = {'charset': 'utf-8', 'Accept-Encoding': 'gzip', 'opensrc': 'list', 'eption': '0a235', 'time': num,
'subsource': '', 'longitude': num1, 'mobileno': '========',忽略账号
'content-type': 'application/x-www-form-urlencoded',
'referer': 'https://servicewechat.com/wx80f809371ae33eda/350/page-frame.html', 'mainsource': '4003',
'userid': '4084302776004076639232125813', 'platform': '4', 'wxapp': '1', 'lang': 'zh',
'latitude': num2, 'citycode': '029', 'accesstoken': '941e812e4db1652fd753af5a81242ac4-0',
'locationtime': '1540817570095',
'User-Agent': 'Mozilla/5.0 (Linux; Android 7.1.2; Redmi 4X Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070336) NetType/WIFI Language/zh_CN Process/appbrand3',
'Content-Length': '184', 'Host': 'mwx.mobike.com', 'Connection': 'Keep-Alive'}
doc1 = (
'biketype=0&speed=0&errMsg=getLocation%3Aok&longitude='+num1+'&verticalAccuracy=0&latitude='+num2+'&accuracy=30&horizontalAccuracy=30&userid=4084302776004076639232125813&citycode=029').encode('ascii')
requests.packages.urllib3.disable_warnings()
r = requests.post(url, data=doc1, headers=doc, verify=False)
doc3 = r.text
return doc3
num=[]
num1=[]
num2=[]
num3=[]
a=0
b=0
xlabel=108.852739
ylabel=34.194159
for x in range(600):
xlabel = xlabel + 0.001
for y in range(600):
b+=1
print('\r{0:2f}%'.format(b/412),end='')
ylabel=ylabel+0.001
doc = mobike(xlabel, ylabel)
doc1 = re.findall(r'distX":\S*?,', doc)
doc2 = re.findall(r'distY":\S*?,', doc)
doc3 = re.findall(r'distance":"\S*?"', doc)
doc4 = re.findall(r'"bikeIds":"\S*?"', doc)
for tag in doc1:
a += 1
tag = tag.replace('distX":', '')
tag = tag.replace(',', '')
num.append(tag)
for tag in doc2:
tag = tag.replace('distY":', '')
tag = tag.replace(',', '')
num1.append(tag)
for tag in doc3:
tag = tag.replace('distance":"', '')
tag = tag.replace('"', '')
num2.append(tag)
for tag in doc4:
tag = tag.replace('"bikeIds":"', '')
tag = tag.replace('"', '')
num3.append(tag)
print(num,num1,num2,num3)
for i in range(a):
fd = open('C:/file/mobike/bikex.txt', 'a+')
fd.write(num[i]+','+num1[i]+','+num2[i]+','+num3[i]+'\n')
num.clear()
num1.clear()
num2.clear()
num3.clear()
a=0
ylabel = 34.194159
##结果（一）
108.8538328,34.19527291,15,0296529592#分别为x,y,距离,车辆id
108.85350429,34.19522746,22,8630345501#
108.85341442,34.19527363,32,8631006108#
108.85409442,34.1953775,40,8630612397#
108.8532906,34.19521682,41,8630905414#
108.8535073,34.19549054,42,8618761638#
108.85325267,34.19528091,46,8630087118#
108.85431808,34.19521107,53,0296535968#
108.85435901,34.19516798,57,8630788171#
108.85342237,34.19471543,57,8630803170#
108.85396464,34.19564681,58,8630674508#
108.85318478,34.19544408,60,8630189207#
108.85440096,34.19529896,62,0296518237#
108.85435404,34.19545009,65,8630036024#
108.85336744,34.19465751,65,8630033074#
108.85342345,34.19573777,70,0290036159#
108.85457573,34.19552874,87,8630499315#
108.85316084,34.19575123,84,8630391691#
108.85368408,34.19598341,91,8630282224#
108.85312989,34.1958193,92,0296537075#
##可视化代码：
<!DOCTYPE html>
<html lang="en">
<head>
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="initial-scale=1.0, user-scalable=no" />
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=5LXmIAWhfm9fuctlUoOwMxmGS4ZUPHOZ"></script>
<script type="text/javascript" src="http://api.map.baidu.com/library/Heatmap/2.0/src/Heatmap_min.js"></script>
<title>热力图功能示例</title>
<style type="text/css">
ul,li{list-style: none;margin:0;padding:0;float:left;}
html{height:100%}
body{height:100%;margin:0px;padding:0px;font-family:"微软雅黑";}
#container{height:100%;width:100%;}
#r-result{width:100%;}
</style>
</head>
<body>
<div id="container"></div>
<div id="r-result" style="display:none">
<input type="button" onclick="openHeatmap();" value="显示热力图"/><input type="button" onclick="closeHeatmap();" value="关闭热力图"/>
</div>
</body>
</html>
<script type="text/javascript">
var map = new BMap.Map("container"); // 创建地图实例
var point = new BMap.Point(108.852739,34.194159);
map.centerAndZoom(point, 13); // 初始化地图，设置中心点坐标和地图级别
map.setCurrentCity("西安"); //设置当前显示城市
map.enableScrollWheelZoom(); // 允许滚轮缩放
var points =[
{"lng":"108.853739","lat":"34.195159","count":"25"},
{"lng":"108.853739","lat":"34.196158","count":"19"},
{"lng":"108.853739","lat":"34.197158","count":"8"},
{"lng":"108.853739","lat":"34.198158","count":"13"},
{"lng":"108.853739","lat":"34.199158","count":"10"},
{"lng":"108.853739","lat":"34.200158","count":"15"},
{"lng":"108.853739","lat":"34.201158","count":"3"},
{"lng":"108.853739","lat":"34.202158","count":"9"},
{"lng":"108.853739","lat":"34.203158","count":"10"},
{"lng":"108.853739","lat":"34.204158","count":"10"},
{"lng":"108.853739","lat":"34.205158","count":"2"},
{"lng":"108.853739","lat":"34.206158","count":"10"},
{"lng":"108.853739","lat":"34.207158","count":"13"},
{"lng":"108.853739","lat":"34.208158","count":"6"},
{"lng":"108.853739","lat":"34.209158","count":"9"},
{"lng":"108.853739","lat":"34.210158","count":"3"},
{"lng":"108.853739","lat":"34.211158","count":"3"},
{"lng":"108.853739","lat":"34.212158","count":"3"},
{"lng":"108.853739","lat":"34.213158","count":"8"},
{"lng":"108.853739","lat":"34.214158","count":"10"},
{"lng":"108.853739","lat":"34.215158","count":"2"},
{"lng":"108.853739","lat":"34.216158","count":"1"},
{"lng":"108.853739","lat":"34.217158","count":"1"},
{"lng":"108.853739","lat":"34.218158","count":"2"},
{"lng":"108.853739","lat":"34.219158","count":"6"},
{"lng":"108.853739","lat":"34.220158","count":"3"},
{"lng":"108.853739","lat":"34.221158","count":"2"},
{"lng":"108.853739","lat":"34.222158","count":"3"},
{"lng":"108.853739","lat":"34.223158","count":"4"},
{"lng":"108.853739","lat":"34.224158","count":"10"},
{"lng":"108.853739","lat":"34.225158","count":"5"},
{"lng":"108.853739","lat":"34.226158","count":"7"},
{"lng":"108.853739","lat":"34.227158","count":"5"},
{"lng":"108.853739","lat":"34.228158","count":"9"},
{"lng":"108.853739","lat":"34.229158","count":"3"},
{"lng":"108.853739","lat":"34.231158","count":"7"},
{"lng":"108.853739","lat":"34.232158","count":"6"},
{"lng":"108.853739","lat":"34.233158","count":"6"},
{"lng":"108.853739","lat":"34.234158","count":"7"},
{"lng":"108.853739","lat":"34.235158","count":"6"},
{"lng":"108.853739","lat":"34.236158","count":"5"},
{"lng":"108.853739","lat":"34.237158","count":"3"},
{"lng":"108.853739","lat":"34.238158","count":"1"},
{"lng":"108.853739","lat":"34.239158","count":"4"},
{"lng":"108.853739","lat":"34.240158","count":"5"},
{"lng":"108.853739","lat":"34.241158","count":"8"},
{"lng":"108.853739","lat":"34.242158","count":"2"},
{"lng":"108.853739","lat":"34.243158","count":"4"}
];//这里面添加经纬度
if(!isSupportCanvas()){
alert('热力图目前只支持有canvas支持的浏览器,您所使用的浏览器不能使用热力图功能~')
}
//详细的参数,可以查看heatmap.js的文档 https://github.com/pa7/heatmap.js/blob/master/README.md
//参数说明如下:
/* visible 热力图是否显示,默认为true
* opacity 热力的透明度,1-100
* radius *图的每个点的半径大小
* gradient {JSON} 热力图的渐变区间 . gradient如下所示
* {
.2:'rgb(0, 255, 255)',
.5:'rgb(0, 110, 255)',
.8:'rgb(100, 0, 255)'
}
其中 key 表示插值的位置, 0~1.
value 为颜色值.
*/
heatmapOverlay = new BMapLib.HeatmapOverlay({"radius":100,"visible":true});
map.addOverlay(heatmapOverlay);
heatmapOverlay.setDataSet({data:points,max:100});
//closeHeatmap();
//判断浏览区是否支持canvas
function isSupportCanvas(){
var elem = document.createElement('canvas');
return !!(elem.getContext && elem.getContext('2d'));
}
function setGradient(){
/*格式如下所示:
{
0:'rgb(102, 255, 0)',
.5:'rgb(255, 170, 0)',
1:'rgb(255, 0, 0)'
}*/
var gradient = {};
var colors = document.querySelectorAll("input[type='color']");
colors = [].slice.call(colors,0);
colors.forEach(function(ele){
gradient[ele.getAttribute("data-key")] = ele.value;
});
heatmapOverlay.setOptions({"gradient":gradient});
}
function openHeatmap(){
heatmapOverlay.show();
}
function closeHeatmap(){
heatmapOverlay.hide();
}
</script>
</body>
</html>
##结果展示