开源基础复习整理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

to tree :len :n
    if :n < 1 [stop]
    forward :len
    left 45
    tree (:len / 2) (:n - 1)
    right 90
    tree (:len / 2) (:n - 1)
    left 45
    back :len
end
//代码如下
import turtle
def tree(length,n):
""" paints a branch of a tree """
    if length < (length/n):
    	return # 边 界 条 件
    turtle.forward(length) # 主 干
    turtle.left(45) # 左 转 分 叉
    tree(length*0.5,length/n) # 左 子 树
    turtle.right(90) # 右 转 分 叉
    tree(length*0.5,length/n) # 右 子 树
    turtle.left(45) # 转 回
    turtle.backward(length) # 回 到 起 点
    return # 返 回

1
2

>>> 3*’un’ + ’ium’
’unununium’

1
2

 word[2:5] # python from position 2 (included) to 5 (excluded)
’tho’

1
2
3
4
5

>>> squares = [1, 4, 9, 16, 25]
>>> squares[-3:] # slicing returns a new list
[9, 16, 25]
>>> squares + [36, 49, 64, 81, 100]
[1, 4, 9, 16, 25, 36, 49, 64, 81, 100]

1
2
3
4
5
6
7
8
9

>>> a = [’a’, ’b’, ’c’]
>>> n = [1, 2, 3]
>>> x = [a, n]
>>> x
[[’a’, ’b’, ’c’], [1, 2, 3]]
>>> x[0]
[’a’, ’b’, ’c’]
>>> x[0][1]
’b’

1
2
3

for i in range(5):
	print(i)
//0,1,2,3,4

1
2
3
4
5
6
7
8

>>> class Complex:
... def __init__(self, realpart, imagpart):
... self.r = realpart
... self.i = imagpart
...
>>> x = Complex(3.0, -4.5)
>>> x.r, x.i
(3.0, -4.5)

1
2
3
4
5
6
7

class A:
def foo(self):
	print("in A::foo")
class B(A):
	pass
b = B()
b.foo()

1
2
3
4
5
6
7
8
9
10
11
12
13

class A:
	def foo(self):
		print("in A::foo")
class B(A):
	def foo(self):
		print("in B::foo")
class C:
	def foo(self):
		print("not subclass of A/B")
b = B()
b.foo()
isinstance(b, B)
isinstance(b, A)

1
2
3
4

inc = lambda x: x + 1
add = lambda x, y: x + y
inc(5)
add(10, 20)

1
2
3
4
5

[i*2 for i in range(10)]
[i*2 for i in range(10) if i % 2 == 0]
colours = ["red", "green", "yellow", "blue"]
things = ["house", "car", "tree"]
combined = [(x,y) for x in colours for y in things]

1
2
3
4

map(lambda x: x**2, range(10)
list(map(lambda x: x** 2, range(10))
filter(lambda x: x % 2 == 0,map(lambda x: x**2, range(10)))
reduce(lambda x, y: x + y,filter(lambda x: x % 2 == 0,map(lambda x: x**2,range(10))))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

>>> t = 12345, 54321, ’hello!’
>>> t[0]
12345
>>> t
(12345, 54321, ’hello!’)
>>> # Tuples may be nested:
... u = t, (1, 2, 3, 4, 5)
>>> u
((12345, 54321, ’hello!’), (1, 2, 3, 4, 5))
>>> # Tuples are immutable:
... t[0] = 88888
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: ’tuple’ object does not support item assignment
>>> # but they can contain mutable objects:
... v = ([1, 2, 3], [3, 2, 1])
>>> v
([1, 2, 3], [3, 2, 1])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

>>> empty = ()
>>> singleton = ’hello’, # <-- note trailing comma
>>> len(empty)
0
>>> len(singleton)
1
>>> singleton
(’hello’,)

>>> x, y, z = t
>>> x, y = y, x
>>> x = (1, 2, 3, [4, 5])
>>> x[3] += [6]
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-34-1b7fadc4e686> in <module>()
----> 1 x[3] += [6]
TypeError: ’tuple’ object does not support item assignment
>>> x
(1, 2, 3, [4, 5, 6])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

>>> tel = {’jack’: 4098, ’sape’: 4139}
>>> tel[’guido’] = 4127
>>> tel
{’sape’: 4139, ’guido’: 4127, ’jack’: 4098}
>>> tel[’jack’]
4098
>>> del tel[’sape’]
>>> tel[’irv’] = 4127
>>> tel
{’guido’: 4127, ’irv’: 4127, ’jack’: 4098}
>>> list(tel.keys())
[’irv’, ’guido’, ’jack’]
>>> sorted(tel.keys())
[’guido’, ’irv’, ’jack’]
>>> ’guido’ in tel
True
>>> ’jack’ not in tel
False

>>> x[(1, 2, 3, [4, 5])] = 6
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-8-7e134042fef1> in <module>()
----> 1 x[(1, 2, 3, [4, 5])] = 6
TypeError: unhashable type: ’list’

1
2
3
4
5
6

>>> dict([(’sape’, 4139),(’guido’, 4127),(’jack’, 4098)])
{’sape’: 4139, ’jack’: 4098, ’guido’: 4127}
>>> {x: x ** 2 for x in (2, 4, 6)}
{2: 4, 4: 16, 6: 36}
>>> dict(sape=4139, guido=4127, jack=4098)
{’sape’: 4139, ’jack’: 4098, ’guido’: 4127}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

>>> basket = {’apple’, ’orange’, ’apple’, ’pear’, ’orange’, ’banana’}
>>> print(basket) # show that duplicates have been removed
{’orange’, ’banana’, ’pear’, ’apple’}
>>> ’orange’ in basket # fast membership testing
True
>>> ’crabgrass’ in basket
False
>>> # Demonstrate set operations on unique letters from two words
...
>>> a = set(’abracadabra’)
>>> b = set(’alacazam’)
>>> a # unique letters in a
{’a’, ’r’, ’b’, ’c’, ’d’}
>>> a - b # letters in a but not in b
{’r’, ’d’, ’b’}
>>> a | b # letters in a or b or both
{’a’, ’c’, ’r’, ’d’, ’b’, ’m’, ’z’, ’l’}
>>> a & b # letters in both a and b
{’a’, ’c’}
>>> a ^ b # letters in a or b but not both
{’r’, ’d’, ’b’, ’m’, ’z’, ’l’}
>>> a = {x for x in ’abracadabra’ if x not in ’abc’}
>>> a
{’r’, ’d’}

1
2
3

User-agent: *
Disallow: / * .php$
Disallow: /images/

1
2
3
4
5
6
7
8

>>> import urllib.robotparser
>>> rp = urllib.robotparser.RobotFileParser()
>>> rp.set_url("http://127.0.0.1:8000/robots.txt")
>>> rp.read()
>>> rp.can_fetch("Baiduspider", "http://127.0.0.1:8000/")
False
>>> rp.can_fetch("Baiduspider", "http://127.0.0.1:8000/article/")
True

1
2
3
4
5
6
7
8
9
10
11

>>> r = requests.get(’https://api.github.com/user’, auth=(’user’, ’pass’))
>>> r.status_code
200
>>> r.headers[’content-type’]
’application/json; charset=utf8’
>>> r.encoding
’utf-8’
>>> r.text
u’{"type":"User"...’
>>> r.json()
{u’private_gists’: 419, u’total_private_repos’: 77, ...}

1
2
3
4
5
6

import requests
r = requests.get("http://127.0.0.1:8000")
print(r.text)
print(r.encoding)
print(r.url)
print(r.status_code)

1
2
3
4
5
6
7

import requests
import json
payload = {’key1’: ’value1’, ’key2’: ’value2’}
r = requests.get(’http://httpbin.org/get’, params=payload)
r.encoding = "UTF-8" //编码问题
result = json.loads(r.text)
print(result[’args’])

1

r = requests.post(’http://httpbin.org/post’, data = {’key’:’value’})

1
2
3
4
5
6
7
8

>>> import re
>>> re.findall(r’\bf[a-z] * ’, ’which foot or hand fell fastest’)
[’foot’, ’fell’, ’fastest’]
>>> re.sub(r’(\b[a-z]+) \1’, r’\1’, ’cat in the the hat’)
’cat in the hat’
//查找所有HTML链接
>>> re.findall("href=’(. * ?)’", html)
[’a.html’, ’b.html’]

1
2
3
4
5
6
7
8
9
10
11

>>> import re
>>> re.match(r’ab * ’, ’abbbb’).group()
’abbbb’
>>> re.match(r’ab * ?’, ’abbbb’).group()
’a’
>>> re.match(r’a\d{3,4}’, ’a1234567’).group()
’a1234’
>>> #使 用|表示 或，\|表 示 字 面 值
>>> import re
>>> re.match(r’(abc)|(def)’, ’abc’).group()
’abc’

1
2
3
4
5
6
7

>>> import re
>>> p = re.compile(’\d+’)
>>> txt = ’12 drummers drumming, 11 pipers piping, 10 lords a-leaping’
>>> p.findall(txt)
[’12’, ’11’, ’10’]
>>> p.match(txt).group()
’12’

1
2
3
4
5
6
7
8
9
10
11
12
13
14

>>> re.match(r’x(\d+) = \1’, ’x123 = 123’).group()
’x123 = 123’
>>> re.match(r’x(\d+) = \1’, ’x123 = 123’).groups()
(’123’,)
>>> p = re.compile(’(a(b)c)d’)
>>> m = p.match(’abcd’)
>>> m.group(0)
’abcd’
>>> m.group(1)
’abc’
>>> m.group(2)
’b’
>>> re.findall(r’(.)\1’, ’明 明 亮 亮 蛋 蛋’)
[’明’, ’亮’, ’蛋’]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

>>> from bs4 import BeautifulSoup
>>> soup = BeautifulSoup(html_doc)
>>> print(soup.prettify())
#html文档
>>> soup.title
<title>The Dormouse’s story</title>
>>> soup.title.name
u’title’
>>> soup.title.string
u’The Dormouse’s story’
>>> soup.title.parent.name
u’head’
>>> soup.p
<p class="title"><b>The Dormouse’s story</b></p>
>>> soup.p[’class’]
u’title’
>>> soup.a
<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
>>> soup.find_all(’a’)
[<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
<a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
>>> soup.find(id="link3")
<a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>

1
2
3
4
5
6

从文档中找到所有 <a> 标签的链接:
for link in soup.find_all(’a’):
	print(link.get(’href’))
# http://example.com/elsie
# http://example.com/lacie
# http://example.com/tillie

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

tag.name
# u’b’
tag[’class’]
# u’boldest’
tag.attrs
# {u’class’: u’boldest’}
tag.string
# u’Extremely bold’
type(tag.string)
# <class ’bs4.element.NavigableString’>
#BeautifulSoup 对象并不是真正的 HTML 或 XML 的 tag， BeautifulSoup 对象包含了一个为“[document]”的特殊属性.name
soup.name
# u’[document]’
#注释部分
markup = "<b><!--Hey, buddy. Want to buy a used parser?--></b>"
soup = BeautifulSoup(markup)
comment = soup.b.string
type(comment)
# <class ’bs4.element.Comment’>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

soup.head
# <head><title>The Dormouse’s story</title></head>
soup.title
# <title>The Dormouse’s story</title>

通过点取属性的方式只能获得当前名字的第一个 tag:
soup.a
# <a class="sister" href="http://example.com/elsie" >

如果想要得到所有的 <a> 标签, 或是通过名字得到比一个 tag 更多的内容的时候, 就需要用到 Searching the tree 中描述的方法, 比如: find_all()
soup.find_all(’a’)
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie"id="link3">Lacie</a>,

tag 的.contents 属性可以将 tag 的子节点以列表的方式输出:
head_tag = soup.head
head_tag
# <head><title>The Dormouse’s story</title></head>
head_tag.contents
[<title>The Dormouse’s story</title>]
title_tag = head_tag.contents[0]
title_tag
# <title>The Dormouse’s story</title>
title_tag.contents
# [u’The Dormouse’s story’]

BeautifulSoup 对象本身一定会包含子节点, 也就是说 <html> 标签也是BeautifulSoup 对象的子节点:
len(soup.contents)
# 1
soup.contents[0].name
# u’html’

通过 tag 的.children 生成器, 可以对 tag 的子节点进行循环:
for child in title_tag.children:
	print(child)
	# The Dormouse’s story
	
继续分析文档树, 每个 tag 或字符串都有父节点: 被包含在某个 tag 中通过.parent 属性来获取某个元素的父节点. 在例子“爱丽丝”的文档中,<head> 标签是 <title> 标签的父节点:
title_tag = soup.title
title_tag
# <title>The Dormouse’s story</title>
title_tag.parent
# <head><title>The Dormouse’s story</title></head>
文档 title 的字符串也有父节点:<title> 标签
title_tag.string.parent
# <title>The Dormouse’s story</title>
文档的顶层节点比如 <html> 的父节点是 BeautifulSoup 对象:
html_tag = soup.html
type(html_tag.parent)
# <class ’bs4.BeautifulSoup’>
BeautifulSoup 对象的.parent 是 None:
print(soup.parent)
# None
通过元素的.parents 属性可以递归得到元素的所有父辈节点, 下面的例子使用了.parents 方法遍历了 <a> 标签到根节点的所有节点.
link = soup.a
link
# <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
for parent in link.parents:
	if parent is None:
		print(parent)
	else:
		print(parent.name)
# p
# body
# html
# [document]
# None

last_a_tag = soup.find("a", id="link3")
last_a_tag
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
last_a_tag.next_sibling
# ’; and they lived at the bottom of a well.’

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

import re
下面例子中找出所有以 b 开头的标签
for tag in soup.find_all(re.compile("^b")):
	print(tag.name)
# body
# b
下面代码找出所有名字中包含”t”的标签:
for tag in soup.find_all(re.compile("t")):
	print(tag.name)
# html
# title

如果传入列表参数,Beautiful Soup 会将与列表中任一元素匹配的内容返回. 下面代码找到文档中所有 <a> 标签和 <b> 标签:
soup.find_all(["a", "b"])
# [<b>The Dormouse’s story</b>,
# <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie"id="link3">Lacie</a>]

True 可以匹配任何值, 下面代码查找到所有的 tag, 但是不会返回字符串
节点
for tag in soup.find_all(True):
	print(tag.name)

1

find_all( name , attrs , recursive , text , **kwargs )

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

soup.find_all("title")
# [<title>The Dormouse’s story</title>]
soup.find_all("p", "title")
# [<p class="title"><b>The Dormouse’s story</b></p>]
soup.find_all("a")
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
soup.find_all(id="link2")
# [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]
import re
soup.find(text=re.compile("sisters"))
# u’Once upon a time there were three little sisters; and their names were\n’

name 参数可以查找所有名字为 name 的 tag, 字符串对象会被自动忽略掉.
soup.find_all("title")
# [<title>The Dormouse’s story</title>]
soup.find_all(id=’link2’)
# [<a class="sister" href="http://example.com/lacie

如果传入 href 参数,Beautiful Soup 会搜索每个 tag 的”href”属性:
soup.find_all(href=re.compile("elsie"))
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

使用多个指定名字的参数可以同时过滤 tag 的多个属性:
soup.find_all(href=re.compile("elsie"), id=’link1’)
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

可以通过 find_all() 方法的 attrs 参数定义一个字典参数来搜索包含特殊属性的 tag:
data_soup.find_all(attrs={"data-foo": "value"})
# [<div data-foo="value">foo!</div>]

按照 CSS 类名搜索 tag 的功能非常实用, 但标识 CSS 类名的关键字class 在 Python 中是保留字, 使用 class 做参数会导致语法错误. 从Beautiful Soup 的 4.1.1 版本开始, 可以通过 class_ 参数搜索有指定CSS 类名的 tag:
soup.find_all("a", class_="sister")
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

tag 的 class 属性是多值属性. 按照 CSS 类名搜索 tag 时, 可以分别搜索tag 中的每个 CSS 类名:
css_soup = BeautifulSoup(’<p class="body strikeout"></p>’)
css_soup.find_all("p", class_="strikeout")
# [<p class="body strikeout"></p>]
css_soup.find_all("p", class_="body")
# [<p class="body strikeout"></p>]

通过 text 参数可以搜搜文档中的字符串内容. 与 name 参数的可选值一样, text 参数接受字符串, 正则表达式, 列表, True . 看例子:
soup.find_all(text="Elsie")
# [u’Elsie’]
soup.find_all(text=["Tillie", "Elsie", "Lacie"])
# [u’Elsie’, u’Lacie’, u’Tillie’]
soup.find_all(text=re.compile("Dormouse"))
[u"The Dormouse’s story", u"The Dormouse’s story"]

虽然 text 参数用于搜索字符串, 还可以与其它参数混合使用来过滤tag.Beautiful Soup 会找到.string 方法与 text 参数值相符的 tag. 下面代码用来搜索内容里面包含“Elsie”的 <a> 标签:
soup.find_all("a", text="Elsie")
#[<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

find_all() 和 find() 只搜索当前节点的所有子节点, 孙子节点等.find_parents() 和 find_parent() 用来搜索当前节点的父辈节点, 搜索方法与普通 tag 的搜索方法相同, 搜索文档搜索文档包含的内容. 我们从一个文档中的一个叶子节点开始:
a_string = soup.find(text="Lacie")
a_string
# u’Lacie’
a_string.find_parents("a")
# [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]
a_string.find_parent("p")
# <p class="story">Once upon a time there were three little sisters; and their names were
# <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a> and
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>;
# and they lived at the bottom of a well.</p>

Beautiful Soup 支持大部分的 CSS 选择器, 在 Tag 或 BeautifulSoup 对象的.select() 方法中传入字符串参数, 即可使用 CSS 选择器的语法找到tag:
soup.select("title")
# [<title>The Dormouse’s story</title>]
soup.select("p nth-of-type(3)")
# [<p class="story">...</p>]

通过 tag 标签逐层查找:
soup.select("body a")
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
soup.select("html head title")
# [<title>The Dormouse’s story</title>]

通过 CSS 的类名查找:
soup.select(".sister")
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
soup.select("[class~=sister]")
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie"

通过 tag 的 id 查找:
soup.select("#link1")
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]
soup.select("a#link2")
# [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

通过是否存在某个属性来查找:
soup.select(’a[href]’)
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

通过属性的值来查找:
soup.select(’a[href="http://example.com/elsie"]’)
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]
soup.select(’a[href^="http://example.com/"]’)
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
# <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
soup.select(’a[href$="tillie"]’)
# [<a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
soup.select(’a[href * =".com/el"]’)
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

1
2
3
4
5
6
7

url = "http://oscar-lab.org/puzzle.html"
txt = requests.get(url).text
soup = bs4.BeatifulSoup(txt, "lxml")
tbody = soup.find("tbody")
for tr in tbody.find_all("tr"):
	for td in tr.find_all("td"):
		#traverse all cells

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

#!/usr/bin/python3

import requests, re, bs4
import subprocess
import glob

prefix = "https://lkml.org"
seed = "%s/lkml/last100/" % prefix

req = requests.get(seed)

soup = bs4.BeautifulSoup(req.text, "lxml")
interested = soup.find_all(class_="mh")
assert(len(interested)) == 1

hrefid = 0
for tr in interested[0].find_all("tr"):
    class_ = tr.get("class")
    if class_ == None or ("c0" not in class_ and "c1" not in class_):
        continue
    tds = tr.find_all('td')
    assert(len(tds) == 3)
    href = tds[1].a['href']
    target = "%d.html" % hrefid
    #subprocess.call(['wget', "%s/%s" % (prefix, href), "-O", target])
    tds[1].a['href'] = target
    tds[2].a['href'] = target
    hrefid += 1

#modify css location
css = soup.find(href=r'/css/message.css')
assert(css != None)
css['href'] = "." + css['href']


handle = open("index.html", "w")
handle.write(soup.prettify())
handle.close()

for html in glob.glob("*.html"):
    handle = open(html)
    soup = bs4.BeautifulSoup(handle.read())
    for script in soup.find_all("script"):
        script.decompose()
    handle.close()
    handle = open(html, "w")
    handle.write(soup.prettify())
    handle.close()

1

open(file, mode=’r’, buffering=-1, encoding=None,errors=None, newline=None,closefd=True, opener=None)

1
2
3
4
5
6
7

handle = open("input.txt", "r")
content = handle.read()
handle.close()
#playing around with content
handle = open("output.txt", "w")
handle.write(content)
handle.close()

1
2
3
4
5
6
7
8
9
10
11
12

>>> import csv
>>> students = [[’name’, ’gender’, ’age’],
... [’zhangsan’, ’male’, 13],
... [’lisi’, ’female’, 14]]
>>> handle = open(’students.csv’, ’w’)
>>> writer = csv.writer(handle)
>>> writer.writerows(students)
>>> handle.close()
>>> handle = open(’students.csv’, ’r’)
>>> reader = csv.reader(handle)
>>> for row in reader:
... print(row)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

import sqlite3
conn = sqlite3.connect(’example.db’)
# or use :memory:
conn_memory = sqlite3.connect(’:memory:’)

c = conn.cursor()
# Create table
c.execute(’’’CREATE TABLE stocks
(date text, trans text, symbol text, qty real, price real)’’’)
# Insert a row of data
c.execute("INSERT INTO stocks VALUES
(’2006-01-05’,’BUY’,’RHAT’,100,35.14)")
# Save (commit) the changes
conn.commit()
# We can also close the connection if we are done with it.
# Just be sure any changes have been committed or they will be lost.
conn.close()

1
2
3
4
5
6
7
8
9
10
11
12
13

# Never do this -- insecure!
symbol = ’RHAT’
c.execute("SELECT * FROM stocks WHERE symbol = ’%s’" % symbol)

# Do this instead
t = (’RHAT’,)
c.execute(’SELECT * FROM stocks WHERE symbol=?’, t)
print(c.fetchone())
# Larger example that inserts many records at a time
purchases = [(’2006-03-28’, ’BUY’, ’IBM’, 1000, 45.00),
			(’2006-04-05’, ’BUY’, ’MSFT’, 1000, 72.00),
			(’2006-04-06’, ’SELL’, ’IBM’, 500, 53.00),]
c.executemany(’INSERT INTO stocks VALUES (?,?,?,?,?)’, purchases)

1
2
3
4
5
6
7
8
9
10
11
12

$ cat a.sql
create table news (id integer, score integer, title text, href text);
insert into news values (1, 8, "hello world", "http://oscar-lab.org");
insert into news values (2, 2, "hello charlie", "http://www.dlut.edu.cn");
$ python3
>>> import sqlite3
>>> conn = sqlite3.connect(’a.db’)
>>> c = conn.cursor()
>>> c.executescript(open(’a.sql’).read())
>>> list(c.execute(’select * from news’))
[(1, 8, ’hello world’, ’http://oscar-lab.org’),
(2, 2, ’hello charlie’, ’http://www.dlut.edu.cn’)]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

>>> conn.row_factory = sqlite3.Row
>>> c = conn.cursor()
>>> c.execute(’select* from stocks’)
<sqlite3.Cursor object at 0x7f4e7dd8fa80>
>>> r = c.fetchone()
>>> type(r)
<class ’sqlite3.Row’>
>>> tuple(r)
(’2006-01-05’, ’BUY’, ’RHAT’, 100.0, 35.14)
>>> len(r)
5
>>> r[2]
’RHAT’

>>> for row in c.execute(’SELECT * FROM stocks ORDER BY price’):
		print(row)
(’2006-01-05’, ’BUY’, ’RHAT’, 100, 35.14)
(’2006-03-28’, ’BUY’, ’IBM’, 1000, 45.0)
(’2006-04-06’, ’SELL’, ’IBM’, 500, 53.0)
(’2006-04-05’, ’BUY’, ’MSFT’, 1000, 72.0)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

{
    "firstName": "John",
    "lastName": "Smith",
    "isAlive": true,
    "age": 25,
    "address": {
        "streetAddress": "21 2nd Street",
        "city": "New York",
        "state": "NY",
        "postalCode": "10021-3100"
    },
    "phoneNumbers": [
        {
            "type": "home",
            "number": "212 555-1234"
        },
        {
            "type": "office",
            "number": "646 555-4567"
        }
    ],
    "children": [],
    "spouse": null
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

>>> import json
>>> json.dumps([’foo’, {’bar’: (’baz’, None, 1.0, 2)}])
’["foo", {"bar": ["baz", null, 1.0, 2]}]’
>>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True))
{"a": 0, "b": 0, "c": 0}
>>> handle = open("output.json", "w")
>>> json.dump({’foo’: 123, ’bar’: ’buz’}, handle)
>>> handle.close()

>>> json.loads(’["foo", {"bar":["baz", null, 1.0, 2]}]’)
[’foo’, {’bar’: [’baz’, None, 1.0, 2]}]
>>> from io import StringIO
>>> handle = open(’output.json’, ‘‘r’’)
>>> b = json.load(handle)
>>> print(b)
{’foo’: 123, ’bar’: ’buz’}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

hjkl左下上右

移动
• gg: goto first line
• G: goto last line
• 0: goto first column
• ˆ: goto first non-blank column
• $: goto last column
• w: next word
• W: next Word
• e: end of word
• E: end of Word
• fa: find a in current line
• Fa: find a in current line backward
• ta: find till before a
• Ta: find till before a backward
单独使用命令
• i: insert
• I: insert at front
• a: append
• A: append after end
• o: append new line
• O: insert new line
命令
• c: Change
• d: Delete
• y: Yank
• gU: Upper case
• gu: Lower case ...
文本对象
• aw: a word
• iw: inner word
• ab: a bracket
• ib: inner bracket
• at: a tag
命令 +移动/文本对象
• dj: delete current and the next lines
• y0: copy to the line head
• ciw: change inner word
• dab: delete a bracket
• gUit: change inner tag to lower case
相对高阶功能
• :%s/re.pattern/replace/g 全局替换
• :g/re.pattern/d 删除匹配行
• :h! E478 Don’t panic
• :h 42

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

#initialization
$ git init
#configuration
$ git config --global user.name "Your Name"
$ git config --global user.email "[email protected]"
# First commit
$ git add example.py
# adds file to the staging area
$ git commit -m "First commit"
# Second commit
$ git add another.py
# adds file to the staging area
$ git commit -m "Second commit"

• By default you are on the "master" branch
• To create a new branch and switch to it:
    $ git branch branchName
    $ git checkout branchName
• Merge changes back to master branch
    $ git checkout master
    $ git merge branchName
• To delete branch
    $ git branch -d branchName

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

import unittest
    class TestStringMethods(unittest.TestCase):
        def test_upper(self):
        	self.assertEqual(’foo’.upper(), ’FOO’)
        def test_isupper(self):
        	self.assertTrue(’FOO’.isupper())
        self.assertFalse(’Foo’.isupper())
        	def test_split(self):
            s = ’hello world’
            self.assertEqual(s.split(), [’hello’, ’world’])
            # check that s.split fails when the separator is not a string
            with self.assertRaises(TypeError):
            	s.split(2)
if __name__ == ’__main__’:
	unittest.main()

1
2
3

test_isupper (__main__.TestStringMethods) ... ok
test_split (__main__.TestStringMethods) ... ok
test_upper (__main__.TestStringMethods) ... ok

1
2
3
4

python -m unittest test_module1 test_module2
python -m unittest test_module.TestClass
python -m unittest test_module.TestClass.test_method
python -m unittest -v test_module

1
2

coverage run --branch foo.py
coverage html

1
2
3
4
5

from flask import Flask
app = Flask(__name__)
@app.route(’/’)
def hello_world():
	return ’Hello, World!’

1
2
3

$ export FLASK_APP=hello.py (run set FLASK_APP=hello.py on Windows)
$ flask run
*Running on http://127.0.0.1:5000/

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

@app.route(’/’)
def index():
	return ’Index Page’
@app.route(’/hello’)
def hello():
	return ’Hello, World’
	
@app.route(’/user/<username>’)
def show_user_profile(username):
    # show the user profile for that user
    return ’User %s’ % username
@app.route(’/post/<int:post_id>’)
def show_post(post_id):
    # show the post with the given id, the id is an integer
    return ’Post %d’ % post_id
    
from flask import request
@app.route(’/login’, methods=[’GET’, ’POST’])
def login():
	if request.method == ’POST’:
		do_the_login()
	else:
		show_the_login_form()

1
2
3
4
5
6
7
8
9

from flask import render_template
@app.route(’/hello/’)
@app.route(’/hello/<name>’)
def hello(name=None):
	return render_template(’hello.html’, name=name)
	
/application.py
/templates
	/hello.html

1
2
3
4
5
6
7
8
9
10
11
12
13
14

@app.route(’/login’, methods=[’POST’, ’GET’])
def login():
    error = None
    if request.method == ’POST’:
        if valid_login(request.form[’username’],
        				request.form[’password’]):
        	return log_the_user_in(request.form[’username’])
    else:
    	error = ’Invalid username/password’
    # the code below is executed if the request method
    # was GET or the credentials were invalid
    return render_template(’login.html’, error=error)
    
searchword = request.args.get(’key’, ’’)

1
2
3
4
5
6
7
8

from flask import abort, redirect, url_for
@app.route(’/’)
def index():
		return redirect(url_for(’login’))
@app.route(’/login’)
def login():
		abort(401)
		this_is_never_executed()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

from flask import Flask, session, redirect, url_for, escape, request
app = Flask(__name__)
@app.route(’/’)
def index():
	if ’username’ in session:
		return ’Logged in as %s’ % escape(session[’username’])
	return ’You are not logged in’
@app.route(’/login’, methods=[’GET’, ’POST’])
def login():
	if request.method == ’POST’:
		session[’username’] = request.form[’username’]
		
>>> import os
>>> os.urandom(24)
’\xfd{H\xe5<\x95\xf9\xe3\x96.5\xd1\x01O<!\xd5\xa2\xa0\x9fR"\xa1\xa8’
Just take that thing and copy/paste it into your code

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from flask import Flask, request, render_template, url_for, redirect
import sqlite3

app = Flask(__name__)

@app.route("/")
def root():
    return redirect("index")
@app.route("/index")
def index():
    return """\
            <html>
                <body>
                    <p> This is the index page</p>
                </body>
            </html>"""

@app.route("/hello/")
@app.route("/hello/<name>")
def hello(name="anonymous"):
    conn = sqlite3.connect("application.db")
    cursor = conn.cursor()
    results = list(cursor.execute("select * from students"))
    conn.close()
    return render_template("hello.html", name=name, results=results)

1
2
3
4
5
6
7
8
9
10
11
12
13
14

import matplotlib.pyplot as plt
import numpy as np
# Data for plotting
t = np.arange(0.0, 2.0, 0.01)
s = 1 + np.sin(2 * np.pi * t)
# Note that using plt.subplots below is equivalent to using
# fig = plt.figure and then ax = fig.add_subplot(111)
fig, ax = plt.subplots()
ax.plot(t, s)
ax.set(xlabel=’time (s)’, ylabel=’voltage (mV)’,
	title=’About as simple as it gets, folks’)
ax.grid()
fig.savefig("test.png")
plt.show()

1
2
3
4
5
6
7
8
9
10
11
12
13

import numpy as np
import matplotlib.pyplot as plt
np.random.seed(19680801)
mu, sigma = 100, 15
x = mu + sigma * np.random.randn(10000)
# the histogram of the data
n, bins, patches = plt.hist(x, 50, normed=1, facecolor=’g’, alpha=0.75)
plt.xlabel(’Smarts’)
plt.ylabel(’Probability’)
plt.title(’Histogram of IQ’)
plt.axis([40, 160, 0, 0.03])
plt.grid(True)
plt.show()

1
2
3
4
5
6
7
8
9
10

import matplotlib.pyplot as plt
# Pie chart, where the slices will be ordered and plotted counter-clockwise:
labels = ’Frogs’, ’Hogs’, ’Dogs’, ’Logs’
sizes = [15, 30, 45, 10]
explode = (0, 0.1, 0, 0) # only "explode" the 2nd slice (i.e. ’Hogs’)
fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct=’%1.1f%%’,
		shadow=True, startangle=90)
ax1.axis(’equal’) # Equal aspect ratio ensures that pie is drawn as a circle.
plt.show()

1
2
3
4
5
6
7
8
9
10

import numpy as np
import matplotlib.pyplot as plt
x = np.linspace(0, 1, 500)
y = np.sin(4 * np.pi * x) * np.exp(-5 * x)
fig, ax = plt.subplots()
ax.fill(x, y, zorder=10)
ax.grid(True, zorder=5)
x = np.linspace(0, 2 * np.pi, 500)
y1 = np.sin(x)
y2 = np.sin(3 * x)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

import numpy as np
import matplotlib.pyplot as plt
# Data for plotting
t = np.arange(0.01, 20.0, 0.01)
# Create figure
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2)
# log y axis
ax1.semilogy(t, np.exp(-t / 5.0))
ax1.set(title=’semilogy’)
ax1.grid()
# log x axis
ax2.semilogx(t, np.sin(2 * np.pi * t))
ax2.set(title=’semilogx’)
ax2.grid()
# log x and y axis
ax3.loglog(t, 20 * np.exp(-t / 10.0), basex=2)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

import numpy as np
import matplotlib.pyplot as plt
# Make some fake data.
a = b = np.arange(0, 3, .02)
c = np.exp(a)
d = c[::-1]
# Create plots with pre-defined labels.
fig, ax = plt.subplots()
ax.plot(a, c, ’k--’, label=’Model length’)
ax.plot(a, d, ’k:’, label=’Data length’)
ax.plot(a, c + d, ’k’, label=’Total message length’)
legend = ax.legend(loc=’upper center’, shadow=True, fontsize=’x-large’)
# Put a nicer background color on the legend.
legend.get_frame().set_facecolor(’#00FFCC’)
plt.show()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

import matplotlib.pyplot as plt
import numpy as np
with plt.xkcd():
    # Based on "Stove Ownership" from XKCD by Randall Monroe
    # http://xkcd.com/418/
    fig = plt.figure()
    ax = fig.add_axes((0.1, 0.2, 0.8, 0.7))
    ax.spines[’right’].set_color(’none’)
    ax.spines[’top’].set_color(’none’)
    plt.xticks([])
    plt.yticks([])
    ax.set_ylim([-30, 10])
    data = np.ones(100)
    data[70:] -= np.arange(30)
    plt.annotate(
        ’THE DAY I REALIZED\nI COULD COOK BACON\nWHENEVER I WANTED’,
        xy=(70, 1), arrowprops=dict(arrowstyle=’->’), xytext=(15, -10))

1
2
3
4
5
6
7
8

--dbpath arg   directory for datafiles - defaults to /data/db
--port arg     specify port number - 27017 by default
--replSet arg  arg is <setname>
--configsvr    declare this is a config db of a cluster;
               default port 27019; default dir /data/configdb
--journal      enable journaling
--nojournal    disable journaling (journaling is on by default for 64 bit)
--logpath arg  log file to send write to instead of stdout - has to be a file, not                  directory

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

CRUD (1): Create
#choose db
>use test
#insert json object into collection
>db.students.insert({"id": 123, "name": "zhangsan", "age": 19})
WriteResult({ "nInserted" : 1 })

CRUD (2): Retrieve
#choose db
>use test
> db.students.find({"id": 123})
{ "_id" : ObjectId("..."), "id" : 123, "name" : "zhangsan", "age" : 19 }

CRUD (3): Update
> db.students.update({"id":123}, {"$inc": {"age":1}})
WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 })
> db.students.find()
{ "_id" : ObjectId("..."), "id" : 123, "name" : "zhangsan", "age" : 20 }

CRUD (4): Delete
> db.students.deleteMany({"id":123})
{ "acknowledged" : true, "deletedCount" : 1 }

1
2
3
4

• via pip
$pip install pymongo
• via distro (e.g., apt)
$sudo apt-get install python-pymongo

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

import pymongo
from pymongo import MongoClient
client = MongoClient()
# Get the sampleDB database
db = client.sampleDB
# equivalently, use db = client[’sampleDB’]
coll = db.sampleCollection
# equivalently, use coll = db[’sampleCollection’]

#CRUD: Create
coll.insert_one({"id": 123, "name": "zhangsan", "age": 18})
coll.insert_one({"id": 124, "name": "lisi", "age": 17})

#CRUD: Retrieve
for entry in coll.find():
	print("id: %d, name: %s, age: %d" %
		(entry[’id’], entry[’name’], entry[’age’])
		
#CRUD: Update
col.update({"age": {"$lt": 20}}, {"$inc": {"age": 1}})
#{u’n’: 1, u’nModified’: 1, u’ok’: 1, ’updatedExisting’: True}
col.update({"age": {"$lt": 20}}, {"$inc": {"age": 1}}, multi=True)
#{u’n’: 2, u’nModified’: 2, u’ok’: 1, ’updatedExisting’: True}

#CRUD: Delete
col.remove({"age": {"$gt": 10}})
#{u’n’: 2, u’ok’: 1}
coll.insert_one({"id": 123, "name": "zhangsan", "age": 18})
coll.insert_one({"id": 124, "name": "lisi", "age": 17})
col.remove({"age": {"$gt": 10}}, multi=False)
#{u’n’: 1, u’ok’: 1}