redis 的使用

setting.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# -*- coding: utf-8 -*-
import scrapy_redis

BOT_NAME = 'CrawlWithRedis'

SPIDER_MODULES = ['CrawlWithRedis.spiders']
NEWSPIDER_MODULE = 'CrawlWithRedis.spiders'

ITEM_PIPELINES = {'CrawlWithRedis.pipelines.CrawlWithRedisPipeline':300}

# Obey robots.txt rules
ROBOTSTXT_OBEY = True

# Redis 数据库设置
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = True
SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderPriorityQueue"
REDIS_URL = None
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379

# MongonDB 设置
MONGODB_HOST = '127.0.0.1'
MONGODB_PORT = 27017
MONGODB_DBNAME = 'XiaoYunKeji'
MONGODB_DOCNAME = 'daomubiji'

items.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

from scrapy import Item, Field

class CrawlWithRedisItem(Item):
bookName = Field()
bookTitle = Field()
chapterNum = Field()
chapterName = Field()
chapterURL = Field()
text = Field()

pipelines.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# -*- coding: utf-8 -*-
# Don't forget to add your pipeline to the ITEM_PIPELINES setting

from scrapy.conf import settings
from CrawlWithRedis.items import CrawlWithRedisItem
import pymongo

class CrawlWithRedisPipeline(object):
def __init__(self):
# 初始化 mongodb 数据库
host = settings['MONGODB_HOST']
port = settings['MONGODB_PORT']
dbName = settings['MONGODB_NAME']
# 连接
db = pymongo.MongoClient(host=host, port=port)
self.post = db[settings['MONGODB_DOCNAME']]

def process_item(self, item, spider):
# 将 item 转换为 dict
item_info = dict(item)
# 插入记录
self.post.insert(item_info)
return item

spiders.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#-*- coding:utf-8 -*-
from scrapy.http import Request
from scrapy.selector import Selector
from scrapy_redis.spiders import RedisSpider
from CrawlWithRedis.items import CrawlWithRedisItem

class spider(RedisSpider):
name = "CrawlWithRedis"
redis_key = 'CrawlWithRedis: start_urls'
start_urls = ['http://daomubiji.com/']

def parse(self, response):
selector = Selector(response)
tables = selector.xpath('//table')
for each_table in tables:
bookName = each_table.xpath('tr/td[@colspan="3"]/center/h2/text()').extract_first()
contents = each_table.xpath('tr/td/a/text()').extract()
urls = each_table.xpath('tr/td/a/@href').extract()
for i in range(len(urls)):
item = CrawlWithRedisItem()
item['bookName'] = bookName
item['chapterURL'] = urls[i]

try:
item['bookTitle'] = contents[i].split(' ')[0]
item['chapterNum'] = contents[i].split(' ')[1]
except Exception as e:
continue
try:
item['chapterName'] = content[i].split(' ')[2]
except Exception as e:
item['chapterName'] = content[i].split(' ')[1][-3:]
yield Request(urls[i], callback=self.parseContent, meta={'item':item})

def parse_Content(self, response):
# 把上面的 item 传递下来
item = response.meta['item']
selector = Selector(response)
contents = selector.xpath('//div[@class="content"]/p/text()').extract()
text = "\n".join(contents)
item['text'] = text
yield item

目前版本 scrapy 1.2 , Python 3.5 运行报错。scrapy github 地址。 暂时未搜索到解决办法, 网上说可能是因为版本问题。改天在 CentOS 上测试下。