爬取百度贴吧尝试

要解析下面的 HTML, 提取到 div 中的文本:

1
<div id="post_content_91765531755" class="d_post_content j_d_post_content  clearfix">            楼主现在iOS10么</div>

定位到 @id 以 post_content 开头并且 @class为 d_post_content j_d_post_content clearfix 的 div。

1
2
3
4
>>> import lxml
>> html = requests.get('http://tieba.baidu.com/p/4609646212')
>>> content = etree.HTML(html.text)
>>> content = content.xpath('//div[starts-with(@id, "post_content") and contains(@class,"d_post_content j_d_post_content  clearfix")]')
  • starts-with(@attr, "xxxx") 函数, 以 xxxx 开头的 attr 属性。
  • contains(@attr, "xxxx") 函数, 精确含有值为 xxxx 的属性。
  • and, 两个函数都为真时, 则返回过滤后的元素。

爬取百度贴吧里面的帖子, 爬取字段为 「回帖日期」、「回帖人」、「回帖内容」:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# -*- coding:utf-8 -*-

from lxml import etree
from multiprocessing.dummy import Pool as ThreadPool
import requests
import json

def spider(url):
# test_url = 'http://tieba.baidu.com/p/4609646212'
html = requests.get(url)
selector = etree.HTML(html.text)
# 获取每个内容块
content_field = selector.xpath('//div[@class="l_post j_l_post l_post_bright "]')
reply = {}
for each_content in content_field:
reply_info = json.loads(each_content.xpath('@data-field')[0])
author = reply_info['author']['user_name']
reply_time = reply_info['content']['date']
content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[starts-with(@id, "post_content") \
and contains(@class,"d_post_content j_d_post_content clearfix")]')
#content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[@class="d_post_content j_d_post_content clearfix"]')
print(author)
print(reply_time)
print(content[0].xpath('string(.)').replace(' ', ''))
print('----------------------------------------------------')
reply['reply_author'] = author
reply['reply_content_time'] = reply_time
reply['reply_content'] = content[0].xpath('string(.)').replace(' ', '')
savetofile(reply)


def savetofile(dict):
f.writelines(u'回帖时间:' + str(dict['reply_content_time']) + "\n")
f.writelines(u'回贴人:' + dict['reply_author'] + "\n")
f.writelines(u'回帖内容:' + dict['reply_content'] + "\n")
f.writelines("\n\n")


if __name__ == '__main__':
pool = ThreadPool(4) # 使用 4 核 cpu
page = []
base_url = 'http://tieba.baidu.com/p/4609646212?pn='
f = open("result.txt", "a", encoding='utf-8') # 将结果写入文件

[page.append(base_url + str(i)) for i in range(1, 21)]

result = pool.map(spider, page)
pool.close()
pool.join()
f.close()

注意, 元素的定位一定要精确, 不然会发生报错。