VB.net 2010 视频教程 VB.net 2010 视频教程 python基础视频教程
SQL Server 2008 视频教程 c#入门经典教程 Visual Basic从门到精通视频教程
当前位置:
首页 > Python基础教程 >

  • 爬虫(十八)：Scrapy框架(五) Scrapy通用爬虫(5)

  1. # -*- coding: utf-8 -*-
  2.  
  3. # Scrapy settings for qd project
  4. #
  5. # For simplicity, this file contains only settings considered important or
  6. # commonly used. You can find more settings consulting the documentation:
  7. #
  8. # https://docs.scrapy.org/en/latest/topics/settings.html
  9. # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
  10. # https://docs.scrapy.org/en/latest/topics/spider-middleware.html
  11.  
  12. BOT_NAME = 'qd'
  13.  
  14. SPIDER_MODULES = ['qd.spiders']
  15. NEWSPIDER_MODULE = 'qd.spiders'
  16.  
  17. DEFAULT_REQUEST_HEADERS = {
  18. 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  19. 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko',
  20. }
  21. # Crawl responsibly by identifying yourself (and your website) on the user-agent
  22. #USER_AGENT = 'qd (+http://www.yourdomain.com)'
  23.  
  24. # Obey robots.txt rules
  25. ROBOTSTXT_OBEY = True
  26.  
  27. # Configure maximum concurrent requests performed by Scrapy (default: 16)
  28. #CONCURRENT_REQUESTS = 32
  29.  
  30. # Configure a delay for requests for the same website (default: 0)
  31. # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
  32. # See also autothrottle settings and docs
  33. #DOWNLOAD_DELAY = 3
  34. # The download delay setting will honor only one of:
  35. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
  36. #CONCURRENT_REQUESTS_PER_IP = 16
  37.  
  38. # Disable cookies (enabled by default)
  39. #COOKIES_ENABLED = False
  40.  
  41. # Disable Telnet Console (enabled by default)
  42. #TELNETCONSOLE_ENABLED = False
  43.  
  44. # Override the default request headers:
  45. #DEFAULT_REQUEST_HEADERS = {
  46. # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  47. # 'Accept-Language': 'en',
  48. #}
  49.  
  50. # Enable or disable spider middlewares
  51. # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
  52. #SPIDER_MIDDLEWARES = {
  53. # 'qd.middlewares.QdSpiderMiddleware': 543,
  54. #}
  55.  
  56. # Enable or disable downloader middlewares
  57. # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
  58. #DOWNLOADER_MIDDLEWARES = {
  59. # 'qd.middlewares.QdDownloaderMiddleware': 543,
  60. #}
  61.  
  62. # Enable or disable extensions
  63. # See https://docs.scrapy.org/en/latest/topics/extensions.html
  64. #EXTENSIONS = {
  65. # 'scrapy.extensions.telnet.TelnetConsole': None,
  66. #}
  67.  
  68. # Configure item pipelines
  69. # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
  70. #ITEM_PIPELINES = {
  71. # 'qd.pipelines.QdPipeline': 300,
  72. #}
  73.  
  74. # Enable and configure the AutoThrottle extension (disabled by default)
  75. # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
  76. #AUTOTHROTTLE_ENABLED = True
  77. # The initial download delay
  78. #AUTOTHROTTLE_START_DELAY = 5
  79. # The maximum download delay to be set in case of high latencies
  80. #AUTOTHROTTLE_MAX_DELAY = 60
  81. # The average number of requests Scrapy should be sending in parallel to
  82. # each remote server
  83. #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
  84. # Enable showing throttling stats for every response received:
  85. #AUTOTHROTTLE_DEBUG = False
  86.  
  87. # Enable and configure HTTP caching (disabled by default)
  88. # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  89. #HTTPCACHE_ENABLED = True
  90. #HTTPCACHE_EXPIRATION_SECS = 0
  91. #HTTPCACHE_DIR = 'httpcache'
  92. #HTTPCACHE_IGNORE_HTTP_CODES = []
  93. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

middlewares.py:



    

  1. 
# -*- coding: utf-8 -*-
    2. 

  2. 
 
    3. 

  3. 
# Define here the models for your spider middleware
    4. 

  4. 
#
    5. 

  5. 
# See documentation in:
    6. 

  6. 
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
    7. 

  7. 
 
    8. 

  8. 
from scrapy import signals
    9. 

  9. 
 
    10. 

  10. 
 
    11. 

  11. 
class qdSpiderMiddleware(object):
    12. 

  12. 
# Not all methods need to be defined. If a method is not defined,
    13. 

  13. 
# scrapy acts as if the spider middleware does not modify the
    14. 

  14. 
# passed objects.
    15. 

  15. 
 
    16. 

  16. 
@classmethod
    17. 

  17. 
def from_crawler(cls, crawler):
    18. 

  18. 
# This method is used by Scrapy to create your spiders.
    19. 

  19. 
s = cls()
    20. 

  20. 
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
    21. 

  21. 
return s
    22. 

  22. 
 
    23. 

  23. 
def process_spider_input(self, response, spider):
    24. 

  24. 
# Called for each response that goes through the spider
    25. 

  25. 
# middleware and into the spider.
    26. 

  26. 
 
    27. 

  27. 
# Should return None or raise an exception.
    28. 

  28. 
return None
    29. 

  29. 
 
    30. 

  30. 
def process_spider_output(self, response, result, spider):
    31. 

  31. 
# Called with the results returned from the Spider, after
    32. 

  32. 
# it has processed the response.
    33. 

  33. 
 
    34. 

  34. 
# Must return an iterable of Request, dict or Item objects.
    35. 

  35. 
for i in result:
    36. 

  36. 
yield i
    37. 

  37. 
 
    38. 

  38. 
def process_spider_exception(self, response, exception, spider):
    39. 

  39. 
# Called when a spider or process_spider_input() method
    40. 

  40. 
# (from other spider middleware) raises an exception.
    41. 

  41. 
 
    42. 

  42. 
# Should return either None or an iterable of Request, dict
    43. 

  43. 
# or Item objects.
    44. 

  44. 
pass
    45. 

  45. 
 
    46. 

  46. 
def
















相关教程