确定 URL
from urllib import requestimport re#定义URLpage=50url="/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn"+str(page)
添加headers并抓取页面代码
try:#定义请求头headers = {'User-Agent': 'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}#定义请求，传入请求头req = request.Request(url, headers=headers)#打开网页response = request.urlopen(req)
面向对象模式
from urllib import requestimport reclass tieba:#初始化def __init__(self):# 定义urlself.url="/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn="# 定义请求头self.headrs={"User-Agent":" Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"}#列表，存储解析后的结果self.stories=[]#下载页面def getPage(self,page_number):try:# 定义请求，传入请求头req=request.Request(self.url+str(page_number),headers=self.headrs)# 打开网页resp=request.urlopen(req)# 打印响应码，解码content=resp.read().decode("utf-8")return contentexcept request.URLError as e:# 打印响应码if hasattr(e, 'code'):print(e.code)# 打印异常原因if hasattr(e, 'reason'):print(e.reason)#解析页面def rexgPage(self,content):# 定义正则表达式# <a rel="noopener" 具体的东西# .*? 匹配没用的数据# (.*?)匹配有用数据分组# \s 空格pattern = pile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</a>')# 匹配htmlitems = re.findall(pattern, content)# 添加解析的内容for i in items:# print("标题:" + i[0] + " 内容:" + i[1])self.stories.append("标题:" + i[0] + " 内容:" + i[1])#显示解析的内容def getContent(self):for i in self.stories:print(i) #创建对象c=tieba()#调用方法c.rexgPage(c.getPage(100))c.getContent()
