北京链家二手房数据分析
首先我们需要将链家二手房的数据抓取下来,用我们上课学的内容,很容易实现(bs4,re,urllib,搞定!):
import re
import csv
from bs4 import BeautifulSoup
from urllib import request
# 成功打开页面时返回页面对象,否则打印错误信息,退出程序
def get_bsobj(url):
## 获取 html 页面
headers = {"Accept-Language":"zh-CN,zh;q=0.9",
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"}
req = request.Request(url,
headers=headers)
## 获取 html 页面
html = request.urlopen(req).read().decode()
bsobj = BeautifulSoup(html, "html5lib")
if bsobj:
return bsobj
else:
print("页面错误")
return None
# 将页面中每一条房屋信息保存为一个字典,将所有的字典保存在列表中,返回列表
def get_house_info_list(url):
house_info_list = []
bsobj = get_bsobj(url)
if not bsobj:
return None
house_list = bsobj.find_all("li", {"class":"clear"})
print(len(house_list))
for house in house_list:
try:
# 标题
title = house.find("div", {"class": "title"}).get_text()
#print(title)
# 获取信息数据(例:加怡名城 | 2室1厅 | 62.48平米 | 西 | 精装),通过“|”符号分割字符串
info = house.find("div", {"class": "houseInfo"}).get_text().split("/")
#print(info)
# 小区(例:加怡名城),strip()去除字符串两边的空格,encode,将字符串编码成 utf-8 格式
block = info[0].strip()
#print(block)
# 房型(例:2室一厅)
house_type = info[1].strip()
#print(house_type)
# 面积大小,保留整数(例:62.48平米,保留整数后为 62)
size_info = info[2].strip()
size = re.findall(r"\d+", size_info)[0]
#print(size)
# 价格,保留整数(例:120.3万,保留整数后为 120)
price_info = house.find("div", {"class": "totalPrice"}).span.get_text()
price = re.findall(r"\d+", price_info)[0]
#print(price)
# 添加到列表中
house_info_list.append({
"title": title,
"price": int(price),
"size": int(size),
"block": block,
"house_type": house_type
})
except IndexError:
pass
#print(house_info_list)
return house_info_list
# 读取前三个页面的房屋信息,将信息保存到 house.csv 文件中
def house(url):
house_info_list = []
# range(10),即前10个子页面 #这里只抓取前10个页面
for i in range(10):
new_url = url + 'pg' + str(i+1)
house_info_list.extend(get_house_info_list(new_url))
print(new_url)
if house_info_list:
# 将数据保存到 house.csv 文件中
with open("./house.csv", "w+") as f:
# writer 对象,修改默认分隔符为 "|"
writer = csv.writer(f, delimiter="|")
for house_info in house_info_list:
title = house_info.get("title")
price = house_info.get("price")
size = house_info.get("size")
block = house_info.get("block")
house_type = house_info.get("house_type")
# 写入一行
try:
writer.writerow([title, int(price), int(size), block, house_type])
print(block, price, size)
except:
continue
house("https://bj.lianjia.com/ershoufang/")
# 简单数据分析案例,太简单了,直接上代码:这里统计了房屋的面积,价格的平均值,标准差,并用直方图来显示结果
# -*- coding: utf-8 -*-
"""
Created on Sun Jun 10 13:07:02 2018
@author: Jun
"""
import numpy
import matplotlib.pyplot as plt
# 读取 house.csv 文件中价格和面积列
price, size = numpy.loadtxt('house.csv', delimiter='|', usecols=(1, 2), unpack=True)
# 求价格和面积的平均值
price_mean = numpy.mean(price)
size_mean = numpy.mean(size)
print("平均价格为:(万元)", price_mean)
print("平均面积为:(平方米)", size_mean)
# 求价格和面积的方差
#price_var = numpy.var(price)
price_var = numpy.std(price)
#size_var = numpy.var(size)
size_var = numpy.std(size)
print("价格的标准差为:(万元)", price_var)
print("面积的标准差为:", size_var)
price, size = numpy.loadtxt('house.csv', delimiter='|', usecols=(1, 2), unpack=True)
plt.figure()
plt.subplot(211)
plt.title("/10000RMB")
plt.hist(price, bins=20)
plt.subplot(212)
plt.xlabel("/m**2")
plt.hist(size, bins=20)
# 运行结果截图