强调内容
BeautifulSoup
灵活又方便的网页解析库，处理高效，支持多种解析器。利用它不用编写正则表达式即可方便地实现网页信息的提取。
解析库
基本使用
html = """<html><head><title>The Dormouse's story</title></head><body><p class="title" name="dromouse"><b>The Dormouse's story</b></p><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><!-- Elsie --></a>,<a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>;and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.prettify()) #自动格式化代码print(soup.title.string)
标签选择器
选择元素
html = """<html><head><title>The Dormouse's story</title></head><body><p class="title" name="dromouse"><b>The Dormouse's story</b></p><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><!-- Elsie --></a>,<a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>;and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.title) # 选择title标签print(type(soup.title)) # <class 'bs4.element.Tag'>print(soup.head) print(soup.p) # 选择段落标签，若有多个只返回第一个结果
获取名称
html = """<html><head><title>The Dormouse's story</title></head><body><p class="title" name="dromouse"><b>The Dormouse's story</b></p><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><!-- Elsie --></a>,<a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>;and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.title.name) # 获取名称
获取属性
html = """<html><head><title>The Dormouse's story</title></head><body><p class="title" name="dromouse"><b>The Dormouse's story</b></p><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><!-- Elsie --></a>,<a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>;and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.p.attrs['name']) # 获取p标签的属性print(soup.p['name']) # 同上
获取内容
html = """<html><head><title>The Dormouse's story</title></head><body><p clss="title" name="dromouse"><b>The Dormouse's story</b></p><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><!-- Elsie --></a>,<a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>;and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.p.string) # 获取内容 .string
嵌套选择
html = """<html><head><title>The Dormouse's story</title></head><body><p class="title" name="dromouse"><b>The Dormouse's story</b></p><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><!-- Elsie --></a>,<a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>;and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.head.title.string)
子节点和子孙节点
html = """<html><head><title>The Dormouse's story</title></head><body><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><span>Elsie</span></a><a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.p.contents) # p标签的子孙
html = """<html><head><title>The Dormouse's story</title></head><body><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><span>Elsie</span></a><a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(type(soup.p.children)) # <class 'list_iterator'> 迭代器print(soup.p.children)for i, child in enumerate(soup.p.children):print(i, child)
html = """<html><head><title>The Dormouse's story</title></head><body><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><span>Elsie</span></a><a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.p.descendants) # 获取所有的子孙节点for i, child in enumerate(soup.p.descendants):print(i, child)
父节点和祖先节点
html = """<html><head><title>The Dormouse's story</title></head><body><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><span>Elsie</span></a><a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.a.parent) # 获取父节点
html = """<html><head><title>The Dormouse's story</title></head><body><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><span>Elsie</span></a><a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(list(enumerate(soup.a.parents))) # 获取所有的祖先节点
兄弟节点
html = """<html><head><title>The Dormouse's story</title></head><body><p class="story">Once upon a time there were three little sisters; and their names were<a href="/elsie" class="sister" id="link1"><span>Elsie</span></a><a href="/lacie" class="sister" id="link2">Lacie</a> and<a href="/tillie" class="sister" id="link3">Tillie</a>and they lived at the bottom of a well.</p><p class="story">...</p>"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(list(enumerate(soup.a.next_siblings))) # 下一个兄弟节点print(list(enumerate(soup.a.previous_siblings))) # 上一个兄弟节点
标准选择器
find_all( name , attrs , recursive , text , **kwargs )
可根据标签名、属性、内容查找文档
name
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.find_all('ul'))print(type(soup.find_all('ul')[0]))
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')for ul in soup.find_all('ul'): # 两层嵌套,通过标签名查找print(ul.find_all('li'))
attrs
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1" name="elements"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.find_all(attrs={'id': 'list-1'})) # 通过标签的属性查找print(soup.find_all(attrs={'name': 'elements'}))
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.find_all(id='list-1')) # 同上，写法更简便print(soup.find_all(class_='panel-heading')) # class_ 要加下划线不然冲突
text
根据文本内容进行选择
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.find_all(text='Foo')) # 做内容匹配还是有用
find( name , attrs , recursive , text , **kwargs )
find返回单个元素，find_all返回所有元素
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.find('ul'))print(type(soup.find('ul')))print(soup.find('page'))
find_parents() find_parent()
find_parents()返回所有祖先节点，find_parent()返回直接父节点。
find_next_siblings() find_next_sibling()
find_next_siblings()返回后面所有兄弟节点，find_next_sibling()返回后面第一个兄弟节点。
find_previous_siblings() find_previous_sibling()
find_previous_siblings()返回前面所有兄弟节点，find_previous_sibling()返回前面第一个兄弟节点。
find_all_next() find_next()
find_all_next()返回节点后所有符合条件的节点, find_next()返回第一个符合条件的节点
find_all_previous() 和 find_previous()
find_all_previous()返回节点后所有符合条件的节点, find_previous()返回第一个符合条件的节点
CSS选择器
通过select()直接传入CSS选择器即可完成选择
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')print(soup.select('.panel .panel-heading')) # 如果是class则加个小数点 .print(soup.select('ul li'))print(soup.select('#list-2 .element')) # #井号代替idprint(type(soup.select('ul')[0]))
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')for ul in soup.select('ul'):print(ul.select('li'))
获取属性
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')for ul in soup.select('ul'):print(ul['id']) print(ul.attrs['id'])
获取内容
html='''<div class="panel"><div class="panel-heading"><h4>Hello</h4></div><div class="panel-body"><ul class="list" id="list-1"><li class="element">Foo</li><li class="element">Bar</li><li class="element">Jay</li></ul><ul class="list list-small" id="list-2"><li class="element">Foo</li><li class="element">Bar</li></ul></div></div>'''from bs4 import BeautifulSoupsoup = BeautifulSoup(html, 'lxml')for li in soup.select('li'):print(li.get_text())
