爬取百度贴吧尝试
— 焉知非鱼百度贴吧
要解析下面的 HTML, 提取到 div 中的文本:
<div id="post_content_91765531755" class="d_post_content j_d_post_content clearfix"> 楼主现在iOS10么</div>
定位到 @id 以 post_content 开头并且 @class为 d_post_content j_d_post_content clearfix 的 div。
>>> import lxml
>> html = requests.get('http://tieba.baidu.com/p/4609646212')
>>> content = etree.HTML(html.text)
>>> content = content.xpath('//div[starts-with(@id, "post_content") and contains(@class,"d_post_content j_d_post_content clearfix")]')
starts-with(@attr, "xxxx")函数, 以 xxxx 开头的 attr 属性。contains(@attr, "xxxx")函数, 精确含有值为 xxxx 的属性。- and, 两个函数都为真时, 则返回过滤后的元素。
爬取百度贴吧里面的帖子, 爬取字段为 「回帖日期」、「回帖人」、「回帖内容」:
# -*- coding:utf-8 -*-
from lxml import etree
from multiprocessing.dummy import Pool as ThreadPool
import requests
import json
def spider(url):
# test_url = 'http://tieba.baidu.com/p/4609646212'
html = requests.get(url)
selector = etree.HTML(html.text)
# 获取每个内容块
content_field = selector.xpath('//div[@class="l_post j_l_post l_post_bright "]')
reply = {}
for each_content in content_field:
reply_info = json.loads(each_content.xpath('@data-field')[0])
author = reply_info['author']['user_name']
reply_time = reply_info['content']['date']
content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[starts-with(@id, "post_content") \
and contains(@class,"d_post_content j_d_post_content clearfix")]')
#content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[@class="d_post_content j_d_post_content clearfix"]')
print(author)
print(reply_time)
print(content[0].xpath('string(.)').replace(' ', ''))
print('----------------------------------------------------')
reply['reply_author'] = author
reply['reply_content_time'] = reply_time
reply['reply_content'] = content[0].xpath('string(.)').replace(' ', '')
savetofile(reply)
def savetofile(dict):
f.writelines(u'回帖时间:' + str(dict['reply_content_time']) + "\n")
f.writelines(u'回贴人:' + dict['reply_author'] + "\n")
f.writelines(u'回帖内容:' + dict['reply_content'] + "\n")
f.writelines("\n\n")
if __name__ == '__main__':
pool = ThreadPool(4) # 使用 4 核 cpu
page = []
base_url = 'http://tieba.baidu.com/p/4609646212?pn='
f = open("result.txt", "a", encoding='utf-8') # 将结果写入文件
[page.append(base_url + str(i)) for i in range(1, 21)]
result = pool.map(spider, page)
pool.close()
pool.join()
f.close()
注意， 元素的定位一定要精确, 不然会发生报错。