增量式爬蟲 Scrapy-Rredis 詳解及案例
- 2020 年 11 月 20 日
- 筆記
- MySQL, Python, Redis, scrapy-redis
1、創建scrapy項目命令
scrapy startproject myproject
2、在項目中創建一個新的spider文件命令：
scrapy genspider mydomain mydomain.com #mydomain為spider文件名，mydomain.com為爬取網站域名
3、運行項目命令
scrapy crawl <spider>
scrapy runspider <spider_file.py> #運行spider第二種方法
4、檢查spider文件有無語法錯誤
scrapy check
5、其他的語法
scrapy crawl <spider> --nolog #運行spider文件 不顯示日誌
scrapy list #列出spider路徑下的spider文件
scrapy fetch <url> #將網頁內容下載下來，然後在終端列印當前返回的內容，相當於 request 和 urllib 方法
scrapy view <url> #將網頁內容保存下來，並在瀏覽器中打開當前網頁內容，直觀呈現要爬取網頁的內容
scrapy shell [url] #打開 scrapy 顯示台，類似ipython，可以用來做測試
scrapy parse <url> [options] #輸出格式化內容
scrapy settings [options] #返回系統設置資訊
scrapy bench #測試電腦當前爬取速度性能
1、以噹噹網為例爬取資訊如下，首先文件的結構目錄如下所示
E:.
│ dangdang_content.sql
│ scrapy.cfg
│
├─.idea
│ │ misc.xml
│ │ modules.xml
│ │ ScrapyRedisPro.iml
│ │ workspace.xml
│ │
│ └─libraries
│ R_User_Library.xml
│
└─ScrapyRedisPro
│ items.py
│ middlewares.py
│ pipelines.py
│ settings.py
│ test.py
│ __init__.py
│
├─.idea
│ │ misc.xml
│ │ modules.xml
│ │ ScrapyRedisPro.iml
│ │ workspace.xml
│ │
│ └─libraries
│ R_User_Library.xml
│
├─spiders
│ │ test.py
│ │ __init__.py
│ │
│ └─__pycache__
│ test.cpython-36.pyc
│ __init__.cpython-36.pyc
│
└─__pycache__
items.cpython-36.pyc
middlewares.cpython-36.pyc
pipelines.cpython-36.pyc
settings.cpython-36.pyc
__init__.cpython-36.pyc
2、mysql資料庫數據結構
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for dangdang_content
-- ----------------------------
DROP TABLE IF EXISTS `dangdang_content`;
CREATE TABLE `dangdang_content` (
`id` int NOT NULL AUTO_INCREMENT,
`b_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`m_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`s_href` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`s_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_img` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_price` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_author` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_publish_date` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_press` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`insert_data` timestamp(0) NULL DEFAULT CURRENT_TIMESTAMP(0) ON UPDATE CURRENT_TIMESTAMP(0),
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
SET FOREIGN_KEY_CHECKS = 1;
1、spider 爬蟲項目文件test.py 如下所示
# -*- coding: utf-8 -*-
import scrapy
from scrapy_redis.spiders import RedisCrawlSpider
from ScrapyRedisPro.items import ScrapyredisproItem
from copy import deepcopy
import urllib
class TestSpider(RedisCrawlSpider):
name = 'test'
# allowed_domains = ['www.baidu.com']
# start_urls = ['//www.baidu.com/']
#調度器隊列名稱
redis_key = 'dangdang'
def parse(self, response):
"""
邏輯分析
1.通過抓取下一頁的鏈接，交給scrapy實現自動翻頁,如果沒有下一頁則爬取完成
2.將本頁面的所有文章url爬下，並交給scrapy進行深入詳情頁的爬取
"""
# 大分類分組
div_list = response.xpath("//div[@class='con flq_body']/div")
for div in div_list:
item = {}
item["b_cate"] = div.xpath("./dl/dt//text()").extract()
item["b_cate"] = [i.strip() for i in item["b_cate"] if len(i.strip()) > 0]
# 中間分類分組
dl_list = div.xpath("./div//dl[@class='inner_dl']")
for dl in dl_list:
item["m_cate"] = dl.xpath("./dt//text()").extract()
item["m_cate"] = [i.strip() for i in item["m_cate"] if len(i.strip()) > 0][0]
# 小分類分組
a_list = dl.xpath("./dd/a")
for a in a_list:
item["s_href"] = a.xpath("./@href").extract_first()
item["s_cate"] = a.xpath("./text()").extract_first()
if item["s_href"] is not None:
# print(item)
yield scrapy.Request(
item["s_href"],
callback=self.parse_book_list,
meta={"item": deepcopy(item)}
)
def parse_book_list(self, response):
"""
將爬蟲爬取的數據送到item中進行序列化
這裡通過ItemLoader載入item
"""
item = response.meta["item"]
li_list = response.xpath("//ul[@class='bigimg']/li")
for li in li_list:
item["book_img"] = li.xpath("./a[@class='pic']/img/@src").extract_first()
if item["book_img"] == "images/model/guan/url_none.png":
item["book_img"] = li.xpath("./a[@class='pic']/img/@data-original").extract_first()
item["book_name"] = li.xpath("./p[@class='name']/a/@title").extract_first()
item["book_desc"] = li.xpath("./p[@class='detail']/text()").extract_first()
item["book_price"] = li.xpath(".//span[@class='search_now_price']/text()").extract_first()
item["book_author"] = li.xpath("./p[@class='search_book_author']/span[1]/a/text()").extract()
item["book_publish_date"] = li.xpath("./p[@class='search_book_author']/span[2]/text()").extract_first()
item["book_press"] = li.xpath("./p[@class='search_book_author']/span[3]/a/text()").extract_first()
# print(item)
yield item
# 下一頁
next_url = response.xpath("//li[@class='next']/a/@href").extract_first()
if next_url is not None:
next_url = urllib.parse.urljoin(response.url, next_url)
yield scrapy.Request(
next_url,
callback=self.parse_book_list,
meta={"item": item}
)
Middlewware.py 中間件文件
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# //doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
from scrapy.http import HtmlResponse
import time
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random
class ScrapyredisproSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn』t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class ScrapyredisproDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
# 攔截到響應對象（下載器傳遞給Spider的響應對象）
# request：響應對象對應的請求對象
# response：攔截到的響應對象
# spider：爬蟲文件中對應的爬蟲類的實例
def process_response(self, request, response, spider):
# 響應對象中存儲頁面數據的篡改
if request.url in ['//news.163.com/domestic/', '//news.163.com/world/', '//news.163.com/air/',
'//war.163.com/']:
spider.bro.get(url=request.url)
js = 'window.scrollTo(0,document.body.scrollHeight)'
spider.bro.execute_script(js)
time.sleep(3) # 一定要給與瀏覽器一定的緩衝載入數據的時間
# 頁面數據就是包含了動態載入出來的新聞數據對應的頁面數據
page_text = spider.bro.page_source
# 篡改響應對象
return HtmlResponse(url=spider.bro.current_url, body=page_text, encoding='utf-8', request=request)
else:
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
#UA池程式碼的編寫（單獨給UA池封裝一個下載中間件的一個類）
#1，導包UserAgentMiddlware類
class RandomUserAgent(UserAgentMiddleware):
def process_request(self, request, spider):
#從列表中隨機抽選出一個ua值
ua = random.choice(user_agent_list)
#ua值進行當前攔截到請求的ua的寫入操作
request.headers.setdefault('User-Agent',ua)
#批量對攔截到的請求進行ip更換
class Proxy(object):
def process_request(self, request, spider):
#對攔截到請求的url進行判斷（協議頭到底是http還是https）
#request.url返回值：//www.xxx.com
h = request.url.split(':')[0] #請求的協議頭
if h == 'https':
ip = random.choice(PROXY_https)
request.meta['proxy'] = '//'+ip
else:
ip = random.choice(PROXY_http)
request.meta['proxy'] = '//' + ip
PROXY_http = [
'153.180.102.104:80',
'195.208.131.189:56055',
]
PROXY_https = [
'120.83.49.90:9000',
'95.189.112.214:35508',
]
user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
"(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]
3、items.py 文件
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# //doc.scrapy.org/en/latest/topics/items.html
import scrapy
class ScrapyredisproItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
b_cate = scrapy.Field()
m_cate = scrapy.Field()
s_href = scrapy.Field()
s_cate = scrapy.Field()
book_img = scrapy.Field()
book_name = scrapy.Field()
book_desc = scrapy.Field()
book_price = scrapy.Field()
book_author = scrapy.Field()
book_publish_date = scrapy.Field()
book_press = scrapy.Field()
4、pipelines.py 管道文件
注意 一般下載方式有兩種，非同步與同步下載方式（下列兩種方式都有）
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: //doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql.cursors
from twisted.enterprise import adbapi
class ScrapyredisproPipeline(object):
# def process_item(self, item, spider):
# return item
# class MysqlTwistedPipeline(object):
'''
非同步機制將數據寫入到mysql資料庫中
'''
# 創建初始化函數，當通過此類創建對象時首先被調用的方法
def __init__(self, dbpool):
self.dbpool = dbpool
# 創建一個靜態方法,靜態方法的載入記憶體優先順序高於init方法，
# 在創建這個類的對之前就已將載入到了記憶體中，所以init這個方法可以調用這個方法產生的對象
@classmethod
# 名稱固定的
def from_settings(cls, settings):
# 先將setting中連接資料庫所需內容取出，構造一個地點
dbparms = dict(
host=settings["MYSQL_HOST"],
port=settings["MYSQL_PORT"],
db=settings["MYSQL_DBNAME"],
user=settings["MYSQL_USER"],
password=settings["MYSQL_PASSWORD"],
charset='utf8',
# 游標設置
cursorclass=pymysql.cursors.DictCursor,
# 設置編碼是否使用Unicode
use_unicode=True
)
# 通過Twisted框架提供的容器連接資料庫,pymysql是資料庫模組名
dbpool = adbapi.ConnectionPool("pymysql",**dbparms)
print("連接成功！")
# 無需直接導入 dbmodule. 只需要告訴 adbapi.ConnectionPool 構造器你用的資料庫模組的名稱比如pymysql.
return cls(dbpool)
def process_item(self, item, spider):
# 使用Twisted非同步的將Item數據插入資料庫
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) # 這裡不往下傳入item,spider，handle_error則不需接受,item,spider)
def do_insert(self, cursor, item):
# 執行具體的插入語句,不需要commit操作,Twisted會自動進行
insert_sql = """
insert into dangdang_content(b_cate, m_cate, s_href, s_cate, book_img, book_name, book_desc, book_price, book_author, book_publish_date, book_press) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
"""
cursor.execute(insert_sql, (item["b_cate"][0], item["m_cate"], item["book_img"], item["s_href"], item["s_cate"], item["book_name"], item["book_desc"], item["book_price"], item["book_author"][0], item["book_publish_date"], item["book_press"]))
print("------------------------------數據插入成功！")
def handle_error(self, failure, item, spider):
# 非同步插入異常
if failure:
print(failure)
# class MysqlTwistedPipeline(object):
# '''
# 同步步機制將數據寫入到mysql資料庫中
# '''
#
# # 創建初始化函數，當通過此類創建對象時首先被調用的方法
# def __init__(self, conn, cursor):
# self.conn = conn
# self.cursor = cursor
#
# # 創建一個靜態方法,靜態方法的載入記憶體優先順序高於init方法，
# # 在創建這個類的對之前就已將載入到了記憶體中，所以init這個方法可以調用這個方法產生的對象
# @classmethod
# # 名稱固定的
# def from_settings(cls, settings):
# # 先將setting中連接資料庫所需內容取出，構造一個地點
# dbparms = dict(
# host=settings["MYSQL_HOST"],
# port=settings["MYSQL_PORT"],
# db=settings["MYSQL_DBNAME"],
# user=settings["MYSQL_USER"],
# password=settings["MYSQL_PASSWORD"],
# charset='utf8',
# # 游標設置
# cursorclass=pymysql.cursors.DictCursor,
# # 設置編碼是否使用Unicode
# use_unicode=True
# )
# conn = pymysql.connect(**dbparms)
# cursor = conn.cursor()
# # 無需直接導入 dbmodule. 只需要告訴 adbapi.ConnectionPool 構造器你用的資料庫模組的名稱比如pymysql.
# return cls(conn, cursor)
#
# def process_item(self, item, spider):
# insert_sql = 'insert into dangdang_content(b_cate[0], m_cate, s_href, s_cate, book_img, book_name, book_desc, book_price, book_author, book_publish_date, book_press) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(item["b_cate"], item["m_cate"], item["book_img"], item["s_href"], item["s_cate"], item["book_name"], item["book_desc"], item["book_price"], item["book_author"][0], item["book_publish_date"], item["book_press"])
# print(insert_sql)
# self.cursor.execute(insert_sql)
# self.conn.commit()
# print("------------------------------數據插入成功！")
#
# def handle_error(self, failure, item, spider):
# # 非同步插入異常
# if failure:
# print(failure)
#
# def close_spider(self, spider):
# """
# 清理
# :param spider:
# :return:
# """
# self.cursor.close()
# self.conn.close()
5、settings.py 配置文件
# -*- coding: utf-8 -*-
# Scrapy settings for ScrapyRedisPro project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# //doc.scrapy.org/en/latest/topics/settings.html
# //doc.scrapy.org/en/latest/topics/downloader-middleware.html
# //doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'ScrapyRedisPro'
SPIDER_MODULES = ['ScrapyRedisPro.spiders']
NEWSPIDER_MODULE = 'ScrapyRedisPro.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'ScrapyRedisPro (+//www.yourdomain.com)'
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' # 偽裝請求載體身份
# Obey robots.txt rules
ROBOTSTXT_OBEY = False #可以忽略或者不遵守robots協議
#只顯示指定類型的日誌資訊
LOG_LEVEL='ERROR'
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See //doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See //doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'ScrapyRedisPro.middlewares.ScrapyredisproSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See //doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
'ScrapyRedisPro.middlewares.ScrapyredisproDownloaderMiddleware': 543,
'ScrapyRedisPro.middlewares.RandomUserAgent': 542,
}
# Enable or disable extensions
# See //doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See //doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'ScrapyRedisPro.pipelines.ScrapyredisproPipeline': 300,
'scrapy_redis.pipelines.RedisPipeline': 400 ,
# 'ScrapyRedisPro.pipelines.MysqlTwistedPipeline': 301,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See //doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See //doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
""" scrapy-redis配置 """
# Enables scheduling storing requests queue in redis.
# 使用scrapy-redis組件自己的調度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 增加了一個去重容器類的配置, 作用使用Redis的set集合來存儲請求的指紋數據, 從而實現請求去重的持久化
# Ensure all spiders share same duplicates filter through redis.
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 配置調度器是否要持久化, 也就是當爬蟲結束了, 要不要清空Redis中請求隊列和去重指紋的set。如果是True, 就表示要持久化存儲, 就不清空數據, 否則清空數據
SCHEDULER_PERSIST = True # 為false Redis關閉了 Redis數據也會被清空
# redis的配置
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379
REDIS_ENCODING ='utf8'
REDIS_PARAMS = {'password':'xhw123'}
MYSQL_HOST = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_DBNAME = 'spidertest'
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'xhw123'
# 增加了一個去重容器類的配置, 作用使用Redis的set集合來存儲請求的指紋數據, 從而實現請求去重的持久化
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 使用scrapy-redis組件自己的調度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 配置調度器是否要持久化, 也就是當爬蟲結束了, 要不要清空Redis中請求隊列和去重指紋的set。如果是True, 就表示要持久化存儲, 就不清空數據, 否則清空數據
SCHEDULER_PERSIST = True
注意 啟動項目後需要在redis的客戶端執行如下命令
127.0.0.1:6379> lpush dangdang //book.dangdang.com/
