增量式爬虫 Scrapy-Rredis 详解及案例
- 2020 年 11 月 20 日
- 笔记
- MySQL, Python, Redis, scrapy-redis
1、创建scrapy项目命令
scrapy startproject myproject
2、在项目中创建一个新的spider文件命令：
scrapy genspider mydomain mydomain.com #mydomain为spider文件名，mydomain.com为爬取网站域名
3、运行项目命令
scrapy crawl <spider>
scrapy runspider <spider_file.py> #运行spider第二种方法
4、检查spider文件有无语法错误
scrapy check
5、其他的语法
scrapy crawl <spider> --nolog #运行spider文件 不显示日志
scrapy list #列出spider路径下的spider文件
scrapy fetch <url> #将网页内容下载下来，然后在终端打印当前返回的内容，相当于 request 和 urllib 方法
scrapy view <url> #将网页内容保存下来，并在浏览器中打开当前网页内容，直观呈现要爬取网页的内容
scrapy shell [url] #打开 scrapy 显示台，类似ipython，可以用来做测试
scrapy parse <url> [options] #输出格式化内容
scrapy settings [options] #返回系统设置信息
scrapy bench #测试电脑当前爬取速度性能
1、以当当网为例爬取信息如下，首先文件的结构目录如下所示
E:.
│ dangdang_content.sql
│ scrapy.cfg
│
├─.idea
│ │ misc.xml
│ │ modules.xml
│ │ ScrapyRedisPro.iml
│ │ workspace.xml
│ │
│ └─libraries
│ R_User_Library.xml
│
└─ScrapyRedisPro
│ items.py
│ middlewares.py
│ pipelines.py
│ settings.py
│ test.py
│ __init__.py
│
├─.idea
│ │ misc.xml
│ │ modules.xml
│ │ ScrapyRedisPro.iml
│ │ workspace.xml
│ │
│ └─libraries
│ R_User_Library.xml
│
├─spiders
│ │ test.py
│ │ __init__.py
│ │
│ └─__pycache__
│ test.cpython-36.pyc
│ __init__.cpython-36.pyc
│
└─__pycache__
items.cpython-36.pyc
middlewares.cpython-36.pyc
pipelines.cpython-36.pyc
settings.cpython-36.pyc
__init__.cpython-36.pyc
2、mysql数据库数据结构
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for dangdang_content
-- ----------------------------
DROP TABLE IF EXISTS `dangdang_content`;
CREATE TABLE `dangdang_content` (
`id` int NOT NULL AUTO_INCREMENT,
`b_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`m_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`s_href` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`s_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_img` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_price` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_author` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_publish_date` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_press` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`insert_data` timestamp(0) NULL DEFAULT CURRENT_TIMESTAMP(0) ON UPDATE CURRENT_TIMESTAMP(0),
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
SET FOREIGN_KEY_CHECKS = 1;
1、spider 爬虫项目文件test.py 如下所示
# -*- coding: utf-8 -*-
import scrapy
from scrapy_redis.spiders import RedisCrawlSpider
from ScrapyRedisPro.items import ScrapyredisproItem
from copy import deepcopy
import urllib
class TestSpider(RedisCrawlSpider):
name = 'test'
# allowed_domains = ['www.baidu.com']
# start_urls = ['//www.baidu.com/']
#调度器队列名称
redis_key = 'dangdang'
def parse(self, response):
"""
逻辑分析
1.通过抓取下一页的链接，交给scrapy实现自动翻页,如果没有下一页则爬取完成
2.将本页面的所有文章url爬下，并交给scrapy进行深入详情页的爬取
"""
# 大分类分组
div_list = response.xpath("//div[@class='con flq_body']/div")
for div in div_list:
item = {}
item["b_cate"] = div.xpath("./dl/dt//text()").extract()
item["b_cate"] = [i.strip() for i in item["b_cate"] if len(i.strip()) > 0]
# 中间分类分组
dl_list = div.xpath("./div//dl[@class='inner_dl']")
for dl in dl_list:
item["m_cate"] = dl.xpath("./dt//text()").extract()
item["m_cate"] = [i.strip() for i in item["m_cate"] if len(i.strip()) > 0][0]
# 小分类分组
a_list = dl.xpath("./dd/a")
for a in a_list:
item["s_href"] = a.xpath("./@href").extract_first()
item["s_cate"] = a.xpath("./text()").extract_first()
if item["s_href"] is not None:
# print(item)
yield scrapy.Request(
item["s_href"],
callback=self.parse_book_list,
meta={"item": deepcopy(item)}
)
def parse_book_list(self, response):
"""
将爬虫爬取的数据送到item中进行序列化
这里通过ItemLoader加载item
"""
item = response.meta["item"]
li_list = response.xpath("//ul[@class='bigimg']/li")
for li in li_list:
item["book_img"] = li.xpath("./a[@class='pic']/img/@src").extract_first()
if item["book_img"] == "images/model/guan/url_none.png":
item["book_img"] = li.xpath("./a[@class='pic']/img/@data-original").extract_first()
item["book_name"] = li.xpath("./p[@class='name']/a/@title").extract_first()
item["book_desc"] = li.xpath("./p[@class='detail']/text()").extract_first()
item["book_price"] = li.xpath(".//span[@class='search_now_price']/text()").extract_first()
item["book_author"] = li.xpath("./p[@class='search_book_author']/span[1]/a/text()").extract()
item["book_publish_date"] = li.xpath("./p[@class='search_book_author']/span[2]/text()").extract_first()
item["book_press"] = li.xpath("./p[@class='search_book_author']/span[3]/a/text()").extract_first()
# print(item)
yield item
# 下一页
next_url = response.xpath("//li[@class='next']/a/@href").extract_first()
if next_url is not None:
next_url = urllib.parse.urljoin(response.url, next_url)
yield scrapy.Request(
next_url,
callback=self.parse_book_list,
meta={"item": item}
)
Middlewware.py 中间件文件
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# //doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
from scrapy.http import HtmlResponse
import time
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random
class ScrapyredisproSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class ScrapyredisproDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
# 拦截到响应对象（下载器传递给Spider的响应对象）
# request：响应对象对应的请求对象
# response：拦截到的响应对象
# spider：爬虫文件中对应的爬虫类的实例
def process_response(self, request, response, spider):
# 响应对象中存储页面数据的篡改
if request.url in ['//news.163.com/domestic/', '//news.163.com/world/', '//news.163.com/air/',
'//war.163.com/']:
spider.bro.get(url=request.url)
js = 'window.scrollTo(0,document.body.scrollHeight)'
spider.bro.execute_script(js)
time.sleep(3) # 一定要给与浏览器一定的缓冲加载数据的时间
# 页面数据就是包含了动态加载出来的新闻数据对应的页面数据
page_text = spider.bro.page_source
# 篡改响应对象
return HtmlResponse(url=spider.bro.current_url, body=page_text, encoding='utf-8', request=request)
else:
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
#UA池代码的编写（单独给UA池封装一个下载中间件的一个类）
#1，导包UserAgentMiddlware类
class RandomUserAgent(UserAgentMiddleware):
def process_request(self, request, spider):
#从列表中随机抽选出一个ua值
ua = random.choice(user_agent_list)
#ua值进行当前拦截到请求的ua的写入操作
request.headers.setdefault('User-Agent',ua)
#批量对拦截到的请求进行ip更换
class Proxy(object):
def process_request(self, request, spider):
#对拦截到请求的url进行判断（协议头到底是http还是https）
#request.url返回值：//www.xxx.com
h = request.url.split(':')[0] #请求的协议头
if h == 'https':
ip = random.choice(PROXY_https)
request.meta['proxy'] = '//'+ip
else:
ip = random.choice(PROXY_http)
request.meta['proxy'] = '//' + ip
PROXY_http = [
'153.180.102.104:80',
'195.208.131.189:56055',
]
PROXY_https = [
'120.83.49.90:9000',
'95.189.112.214:35508',
]
user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
"(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]
3、items.py 文件
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# //doc.scrapy.org/en/latest/topics/items.html
import scrapy
class ScrapyredisproItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
b_cate = scrapy.Field()
m_cate = scrapy.Field()
s_href = scrapy.Field()
s_cate = scrapy.Field()
book_img = scrapy.Field()
book_name = scrapy.Field()
book_desc = scrapy.Field()
book_price = scrapy.Field()
book_author = scrapy.Field()
book_publish_date = scrapy.Field()
book_press = scrapy.Field()
4、pipelines.py 管道文件
注意 一般下载方式有两种，异步与同步下载方式（下列两种方式都有）
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: //doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql.cursors
from twisted.enterprise import adbapi
class ScrapyredisproPipeline(object):
# def process_item(self, item, spider):
# return item
# class MysqlTwistedPipeline(object):
'''
异步机制将数据写入到mysql数据库中
'''
# 创建初始化函数，当通过此类创建对象时首先被调用的方法
def __init__(self, dbpool):
self.dbpool = dbpool
# 创建一个静态方法,静态方法的加载内存优先级高于init方法，
# 在创建这个类的对之前就已将加载到了内存中，所以init这个方法可以调用这个方法产生的对象
@classmethod
# 名称固定的
def from_settings(cls, settings):
# 先将setting中连接数据库所需内容取出，构造一个地点
dbparms = dict(
host=settings["MYSQL_HOST"],
port=settings["MYSQL_PORT"],
db=settings["MYSQL_DBNAME"],
user=settings["MYSQL_USER"],
password=settings["MYSQL_PASSWORD"],
charset='utf8',
# 游标设置
cursorclass=pymysql.cursors.DictCursor,
# 设置编码是否使用Unicode
use_unicode=True
)
# 通过Twisted框架提供的容器连接数据库,pymysql是数据库模块名
dbpool = adbapi.ConnectionPool("pymysql",**dbparms)
print("连接成功！")
# 无需直接导入 dbmodule. 只需要告诉 adbapi.ConnectionPool 构造器你用的数据库模块的名称比如pymysql.
return cls(dbpool)
def process_item(self, item, spider):
# 使用Twisted异步的将Item数据插入数据库
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) # 这里不往下传入item,spider，handle_error则不需接受,item,spider)
def do_insert(self, cursor, item):
# 执行具体的插入语句,不需要commit操作,Twisted会自动进行
insert_sql = """
insert into dangdang_content(b_cate, m_cate, s_href, s_cate, book_img, book_name, book_desc, book_price, book_author, book_publish_date, book_press) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
"""
cursor.execute(insert_sql, (item["b_cate"][0], item["m_cate"], item["book_img"], item["s_href"], item["s_cate"], item["book_name"], item["book_desc"], item["book_price"], item["book_author"][0], item["book_publish_date"], item["book_press"]))
print("------------------------------数据插入成功！")
def handle_error(self, failure, item, spider):
# 异步插入异常
if failure:
print(failure)
# class MysqlTwistedPipeline(object):
# '''
# 同步步机制将数据写入到mysql数据库中
# '''
#
# # 创建初始化函数，当通过此类创建对象时首先被调用的方法
# def __init__(self, conn, cursor):
# self.conn = conn
# self.cursor = cursor
#
# # 创建一个静态方法,静态方法的加载内存优先级高于init方法，
# # 在创建这个类的对之前就已将加载到了内存中，所以init这个方法可以调用这个方法产生的对象
# @classmethod
# # 名称固定的
# def from_settings(cls, settings):
# # 先将setting中连接数据库所需内容取出，构造一个地点
# dbparms = dict(
# host=settings["MYSQL_HOST"],
# port=settings["MYSQL_PORT"],
# db=settings["MYSQL_DBNAME"],
# user=settings["MYSQL_USER"],
# password=settings["MYSQL_PASSWORD"],
# charset='utf8',
# # 游标设置
# cursorclass=pymysql.cursors.DictCursor,
# # 设置编码是否使用Unicode
# use_unicode=True
# )
# conn = pymysql.connect(**dbparms)
# cursor = conn.cursor()
# # 无需直接导入 dbmodule. 只需要告诉 adbapi.ConnectionPool 构造器你用的数据库模块的名称比如pymysql.
# return cls(conn, cursor)
#
# def process_item(self, item, spider):
# insert_sql = 'insert into dangdang_content(b_cate[0], m_cate, s_href, s_cate, book_img, book_name, book_desc, book_price, book_author, book_publish_date, book_press) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(item["b_cate"], item["m_cate"], item["book_img"], item["s_href"], item["s_cate"], item["book_name"], item["book_desc"], item["book_price"], item["book_author"][0], item["book_publish_date"], item["book_press"])
# print(insert_sql)
# self.cursor.execute(insert_sql)
# self.conn.commit()
# print("------------------------------数据插入成功！")
#
# def handle_error(self, failure, item, spider):
# # 异步插入异常
# if failure:
# print(failure)
#
# def close_spider(self, spider):
# """
# 清理
# :param spider:
# :return:
# """
# self.cursor.close()
# self.conn.close()
5、settings.py 配置文件
# -*- coding: utf-8 -*-
# Scrapy settings for ScrapyRedisPro project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# //doc.scrapy.org/en/latest/topics/settings.html
# //doc.scrapy.org/en/latest/topics/downloader-middleware.html
# //doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'ScrapyRedisPro'
SPIDER_MODULES = ['ScrapyRedisPro.spiders']
NEWSPIDER_MODULE = 'ScrapyRedisPro.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'ScrapyRedisPro (+//www.yourdomain.com)'
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' # 伪装请求载体身份
# Obey robots.txt rules
ROBOTSTXT_OBEY = False #可以忽略或者不遵守robots协议
#只显示指定类型的日志信息
LOG_LEVEL='ERROR'
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See //doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See //doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'ScrapyRedisPro.middlewares.ScrapyredisproSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See //doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
'ScrapyRedisPro.middlewares.ScrapyredisproDownloaderMiddleware': 543,
'ScrapyRedisPro.middlewares.RandomUserAgent': 542,
}
# Enable or disable extensions
# See //doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See //doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'ScrapyRedisPro.pipelines.ScrapyredisproPipeline': 300,
'scrapy_redis.pipelines.RedisPipeline': 400 ,
# 'ScrapyRedisPro.pipelines.MysqlTwistedPipeline': 301,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See //doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See //doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
""" scrapy-redis配置 """
# Enables scheduling storing requests queue in redis.
# 使用scrapy-redis组件自己的调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 增加了一个去重容器类的配置, 作用使用Redis的set集合来存储请求的指纹数据, 从而实现请求去重的持久化
# Ensure all spiders share same duplicates filter through redis.
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 配置调度器是否要持久化, 也就是当爬虫结束了, 要不要清空Redis中请求队列和去重指纹的set。如果是True, 就表示要持久化存储, 就不清空数据, 否则清空数据
SCHEDULER_PERSIST = True # 为false Redis关闭了 Redis数据也会被清空
# redis的配置
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379
REDIS_ENCODING ='utf8'
REDIS_PARAMS = {'password':'xhw123'}
MYSQL_HOST = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_DBNAME = 'spidertest'
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'xhw123'
# 增加了一个去重容器类的配置, 作用使用Redis的set集合来存储请求的指纹数据, 从而实现请求去重的持久化
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 使用scrapy-redis组件自己的调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 配置调度器是否要持久化, 也就是当爬虫结束了, 要不要清空Redis中请求队列和去重指纹的set。如果是True, 就表示要持久化存储, 就不清空数据, 否则清空数据
SCHEDULER_PERSIST = True
注意 启动项目后需要在redis的客户端执行如下命令
127.0.0.1:6379> lpush dangdang //book.dangdang.com/
