【Class 20】【实例】python 爬虫简单案例实现二---将数据保存为CSV文件

摘抄一位网友的写入和读取csv的代码：

#输出数据写入CSV文件import csvdata = [("Mike", "male", 24),("Lee", "male", 26),("Joy", "female", 22)]#Python3.4以后的新方式，解决空行问题with open('d://write.csv', 'w', newline='') as csv_file:csv_writer = csv.writer(csv_file)for list in data:print(list)csv_writer.writerow(list)

#读取csv文件内容import csvlist = []reader = csv.reader(open("d://demo.csv"))#csv中有三列数据，遍历读取时使用三个变量分别对应for title, year, director in reader:list.append(year)print(title, "; ", year , "; ", director)print(list)

我们在前面 class 19 中，将保存为csv的功能加入进去：

保存csv的代码如下

# 保存数据为csv格式def __SaveCSV(self, anchors):with open('d://PandaTV_data.csv', 'w', newline='') as csv_file:csv_writer = csv.writer(csv_file)for value in anchors:csv_writer.writerow([value['name'], value['number']])

完整代码如下:

# Ciellee -02-24 22:00# 爬虫前奏：# 明确目的: 获取熊猫TV 英雄联盟主播人气排行榜# 找到数据对应的网页：https://www.panda.tv/cate/lol?pdt=1.c_lol.psbar-ca0.0.29u3363v9n8# 分析网页的结构，找到数据所在的标签位置: video-info { video-nickname, video-number }# 待分析网页数据# <div class="video-info">#<span class="video-title" title="LPL春季赛RW vs EDG">LPL春季赛RW vs EDG</span>#<span class="video-nickname" title="LPL熊猫官方直播2台"><i></i>LPL熊猫官方直播2台</span>#<span class="video-number"><i class="ricon ricon-eye"></i>678.9万</span># </div># 模拟HTTP请求，向服务器发送个请求，获取到服务器返回给我们的HTML# 用正则表达式提取我们要的数据 video-nickname,video-numberfrom urllib import requestimport reimport csvclass Spider():url = 'https://www.panda.tv/cate/lol?pdt=1.c_lol.psbar-ca0.0.29u3363v9n8'# 匹配字符串 \s\S: 匹配所有字符串 *:匹配无限多个 ?:采用非贪婪模式root_pattern = '<div class="video-info">([\s\S]*?)</div>'name_pattern = '<span class="video-nickname" title="([\s\S]*?)">'number_pattern = '="video-number"><i class="ricon ricon-eye"></i>([\s\S]*?)</span>'# 私有方法，访问网页def __fetch_content(self):r = request.urlopen(Spider.url)# 类型为字节码，byteshtmls = r.read()#print(type(htmls)) # 将字节码转换为字符串文本htmls=str(htmls, encoding='utf-8')#print(type(htmls)) return htmls# 分析文本def __analysis(self,htmls):root_html = re.findall(Spider.root_pattern, htmls)#print(type(root_html))#print(root_html[0])# 新建一个空字典anchors = []for html_tmp in root_html:name = re.findall(Spider.name_pattern, html_tmp)number = re.findall(Spider.number_pattern, html_tmp)# 将name 和 number 拼成一个字典anchor = {'name':name, 'number':number}anchors.append(anchor)#print(anchors[0])return anchors# 对数据进行修饰# strip() 方法用于移除字符串头尾指定的字符（默认为空格或换行符）或字符序列def __refine(self, anchors):lam = lambda anchors:{'name':anchors['name'][0].strip(),'number':anchors['number'][0]}return map(lam, anchors)# 对数据进行分析，排序def __sort(self, anchors):anchors = sorted(anchors, key=self.__sort_seed, reverse=True)return anchors# 排序的key, 将number从字符串提取为整数def __sort_seed(self, anchor):r = re.findall('\d*\.*\d*', anchor['number'])number = float(r[0])#print(number)if '万' in anchor['number']:number *= 10000.0#print(number)return number# 显示数据def __show(self, anchors):#for anchor in anchors:for rank in range(0, len(anchors)):print('第' + str(rank+1) + '名' + ' : '+ anchors[rank]['name'] +' ------ '+anchors[rank]['number'] + '人')# 保存数据为csv格式def __SaveCSV(self, anchors):with open('d://PandaTV_data.csv', 'w', newline='') as csv_file:csv_writer = csv.writer(csv_file)for value in anchors:csv_writer.writerow([value['name'], value['number']]) # 用户接口def go(self):# 访问网页htmls = self.__fetch_content()# 解析数据anchors = self.__analysis( htmls )# 对数据进行修饰anchors = list( self.__refine(anchors) )# 分析数据，排序anchors = self.__sort(anchors)# 保存数据为csv格式self.__SaveCSV(anchors)# 显示打印数据#self.__show(anchors)#print( anchors )spider = Spider()spider.go()

结果如下: