python3 网络爬虫 实例1

• 2019 年 12 月 13 日
• 筆記

scrapy

pip install scrapy pip install pyOpenSSL pip install cryptography pip install CFFI pip install lxml pip install cssselect pip install Twisted

创建爬虫项目

scrapy startproject zhipinSpider

生成爬虫

scrapy genspider job_position "zhipin.com"

image.png

目录结构： items.py : pipelines.py:处理爬取的内容 settings.py :配置文件

先调试数据

scrapy shell – s USER_AGENT="xx" https://www.zhipin.com/c101280100/h101280100/

让scrapy伪装成浏览器

XPath语法

/ 匹配根节点 // 任意节点 . 当前节点 .. 父节点 @ 属性 //div[@title="xxx"]/div

extract提取节点内容

image.png

CSS匹配

image.png

items.py

import scrapy

class ZhipinspiderItem(scrapy.Item): # 工作名称 title = scrapy.Field() # 工资 salary = scrapy.Field() # 招聘公司 company = scrapy.Field() # 工作详细链接 url = scrapy.Field() # 工作地点 work_addr = scrapy.Field() # 行业 industry = scrapy.Field() # 公司规模 company_size = scrapy.Field() # 招聘人 recruiter = scrapy.Field() # 发布时间 publish_date = scrapy.Field()

job_spider.py

import scrapy from ZhipinSpider.items import ZhipinspiderItem

class JobPositionSpider(scrapy.Spider): # 定义该Spider的名字 name = 'job_position' # 定义该Spider允许爬取的域名 allowed_domains = ['zhipin.com'] # 定义该Spider爬取的首页列表 start_urls = ['https://www.zhipin.com/c101280100/h_101280100/']

# 该方法负责提取response所包含的信息  # response代表下载器从start_urls中每个URL下载得到的响应  def parse(self, response):      # 遍历页面上所有//div[@class="job-primary"]节点      for job_primary in response.xpath('//div[@class="job-primary"]'):          item = ZhipinspiderItem()          # 匹配//div[@class="job-primary"]节点下/div[@class="info-primary"]节点          # 也就是匹配到包含工作信息的<div.../>元素          info_primary = job_primary.xpath('./div[@class="info-primary"]')          item['title'] = info_primary.xpath('./h3/a/div[@class="job-title"]/text()').extract_first()          item['salary'] = info_primary.xpath('./h3/a/span[@class="red"]/text()').extract_first()          item['work_addr'] = info_primary.xpath('./p/text()').extract_first()          item['url'] = info_primary.xpath('./h3/a/@href').extract_first()          # 匹配//div[@class="job-primary"]节点下./div[@class="info-company"]节点下          # 的/div[@class="company-text"]的节点          # 也就是匹配到包含公司信息的<div.../>元素          company_text = job_primary.xpath('./div[@class="info-company"]' +              '/div[@class="company-text"]')          item['company'] = company_text.xpath('./h3/a/text()').extract_first()          company_info = company_text.xpath('./p/text()').extract()          if company_info and len(company_info) > 0:              item['industry'] = company_info[0]          if company_info and len(company_info) > 2:              item['company_size'] = company_info[2]          # 匹配//div[@class="job-primary"]节点下./div[@class="info-publis"]节点下          # 也就是匹配到包含发布人信息的<div.../>元素          info_publis = job_primary.xpath('./div[@class="info-publis"]')          item['recruiter'] = info_publis.xpath('./h3/text()').extract_first()          item['publish_date'] = info_publis.xpath('./p/text()').extract_first()          yield item        # 解析下一页的链接      new_links = response.xpath('//div[@class="page"]/a[@class="next"]/@href').extract()      if new_links and len(new_links) > 0:          # 获取下一页的链接          new_link = new_links[0]          # 再次发送请求获取下一页数据          yield scrapy.Request("https://www.zhipin.com" + new_link, callback=self.parse)

pipelines.py

class ZhipinspiderPipeline(object): def process_item(self, item, spider): print("工作:" , item['title']) print("工资:" , item['salary']) print("工作地点:" , item['work_addr']) print("详情链接:" , item['url'])

    print("公司:" , item['company'])      print("行业:" , item['industry'])      print("公司规模:" , item['company_size'])        print("招聘人:" , item['recruiter'])      print("发布日期:" , item['publish_date'])

settings.py

–– coding: utf-8 ––

Scrapy settings for ZhipinSpider project

For simplicity, this file contains only settings considered important or

commonly used. You can find more settings consulting the documentation:

https://doc.scrapy.org/en/latest/topics/settings.html

https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'ZhipinSpider'

SPIDER_MODULES = ['ZhipinSpider.spiders'] NEWSPIDER_MODULE = 'ZhipinSpider.spiders'

Crawl responsibly by identifying yourself (and your website) on the user-agent

USER_AGENT = 'ZhipinSpider (+http://www.yourdomain.com)'

Obey robots.txt rules

ROBOTSTXT_OBEY = True

Configure maximum concurrent requests performed by Scrapy (default: 16)

CONCURRENT_REQUESTS = 32

Configure a delay for requests for the same website (default: 0)

See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay

See also autothrottle settings and docs

DOWNLOAD_DELAY = 3

The download delay setting will honor only one of:

CONCURRENT_REQUESTS_PER_DOMAIN = 16

CONCURRENT_REQUESTS_PER_IP = 16

Disable cookies (enabled by default)

COOKIES_ENABLED = False

Disable Telnet Console (enabled by default)

TELNETCONSOLE_ENABLED = False

Override the default request headers:

配置默认的请求头

DEFAULT_REQUEST_HEADERS = { "User-Agent" : "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0", 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8' }

Enable or disable spider middlewares

See https://doc.scrapy.org/en/latest/topics/spider-middleware.html

SPIDER_MIDDLEWARES = {

'ZhipinSpider.middlewares.ZhipinspiderSpiderMiddleware': 543,

}

Enable or disable downloader middlewares

See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

DOWNLOADER_MIDDLEWARES = {

'ZhipinSpider.middlewares.ZhipinspiderDownloaderMiddleware': 543,

}

Enable or disable extensions

See https://doc.scrapy.org/en/latest/topics/extensions.html

EXTENSIONS = {

'scrapy.extensions.telnet.TelnetConsole': None,

}

Configure item pipelines

See https://doc.scrapy.org/en/latest/topics/item-pipeline.html

配置使用Pipeline

ITEM_PIPELINES = { 'ZhipinSpider.pipelines.ZhipinspiderPipeline': 300, }

Enable and configure the AutoThrottle extension (disabled by default)

See https://doc.scrapy.org/en/latest/topics/autothrottle.html

AUTOTHROTTLE_ENABLED = True

The initial download delay

AUTOTHROTTLE_START_DELAY = 5

The maximum download delay to be set in case of high latencies

AUTOTHROTTLE_MAX_DELAY = 60

The average number of requests Scrapy should be sending in parallel to

each remote server

AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0

Enable showing throttling stats for every response received:

AUTOTHROTTLE_DEBUG = False

Enable and configure HTTP caching (disabled by default)

See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings

HTTPCACHE_ENABLED = True

HTTPCACHE_EXPIRATION_SECS = 0

HTTPCACHE_DIR = 'httpcache'

HTTPCACHE_IGNORE_HTTP_CODES = []

HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

启动

scrapy crawl job_position

存入数据库：pipelines.py

导入访问MySQL的模块

import mysql.connector

class ZhipinspiderPipeline(object): # 定义构造器，初始化要写入的文件 def init(self): self.conn = mysql.connector.connect(user='root', password='32147', host='localhost', port='3306', database='python', use_unicode=True) self.cur = self.conn.cursor() # 重写close_spider回调方法，用于关闭数据库资源 def close_spider(self, spider): print('———-关闭数据库资源———–') # 关闭游标 self.cur.close() # 关闭连接 self.conn.close() def process_item(self, item, spider): self.cur.execute("INSERT INTO job_inf VALUES(null, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (item['title'], item['salary'], item['company'], item['url'], item['work_addr'], item['industry'], item.get('company_size'), item['recruiter'], item['publish_date'])) self.conn.commit()

处理反爬虫

更改IP地址：middlewares.py

image.png

禁用cookie:settings.py

COOKIES_ENABLED=False

不遵守爬虫规则

image.png

设置访问频率

image.png

image.png

登录selenium