VB.net 2010 视频教程 VB.net 2010 视频教程 python基础视频教程
SQL Server 2008 视频教程 c#入门经典教程 Visual Basic从门到精通视频教程
当前位置:
首页 > Python基础教程 >

  • 爬虫(十七)：Scrapy框架(四) 对接selenium爬取京东商品数据(6)

mongo_url
  • self.mongo_db = mongo_db
  • self.collection = collection
  •  
  • @classmethod
  • #from_crawler是一个类方法，由 @classmethod标识，是一种依赖注入的方式，它的参数就是crawler
  • #通过crawler我们可以拿到全局配置的每个配置信息，在全局配置settings.py中的配置项都可以取到。
  • #所以这个方法的定义主要是用来获取settings.py中的配置信息
  • def from_crawler(cls,crawler):
  • return cls(
  • mongo_url=crawler.settings.get('MONGO_URL'),
  • mongo_db = crawler.settings.get('MONGO_DB'),
  • collection = crawler.settings.get('COLLECTION')
  • )
  •  
  • def open_spider(self,spider):
  • self.client = pymongo.MongoClient(self.mongo_url)
  • self.db = self.client[self.mongo_db]
  •  
  • def process_item(self,item, spider):
  • # name = item.__class__.collection
  • name = self.collection
  • self.db[name].insert(dict(item))
  • return item
  •  
  • def close_spider(self,spider):
  • self.client.close()

    • settings.py:

    
	
      
		
    1. 
			# -*- coding: utf-8 -*-
      2. 
		
    2. 
			 
      3. 
		
    3. 
			# Scrapy settings for scrapyseleniumtest project
      4. 
		
    4. 
			#
      5. 
		
    5. 
			# For simplicity, this file contains only settings considered important or
      6. 
		
    6. 
			# commonly used. You can find more settings consulting the documentation:
      7. 
		
    7. 
			#
      8. 
		
    8. 
			# https://docs.scrapy.org/en/latest/topics/settings.html
      9. 
		
    9. 
			# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
      10. 
		
    10. 
			# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
      11. 
		
    11. 
			 
      12. 
		
    12. 
			BOT_NAME = 'scrapyseleniumtest'
      13. 
		
    13. 
			 
      14. 
		
    14. 
			SPIDER_MODULES = ['scrapyseleniumtest.spiders']
      15. 
		
    15. 
			NEWSPIDER_MODULE = 'scrapyseleniumtest.spiders'
      16. 
		
    16. 
			 
      17. 
		
    17. 
			 
      18. 
		
    18. 
			# Crawl responsibly by identifying yourself (and your website) on the user-agent
      19. 
		
    19. 
			#USER_AGENT = 'scrapyseleniumtest (+http://www.yourdomain.com)'
      20. 
		
    20. 
			 
      21. 
		
    21. 
			# Obey robots.txt rules
      22. 
		
    22. 
			ROBOTSTXT_OBEY = False
      23. 
		
    23. 
			 
      24. 
		
    24. 
			# Configure maximum concurrent requests performed by Scrapy (default: 16)
      25. 
		
    25. 
			#CONCURRENT_REQUESTS = 32
      26. 
		
    26. 
			 
      27. 
		
    27. 
			# Configure a delay for requests for the same website (default: 0)
      28. 
		
    28. 
			# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
      29. 
		
    29. 
			# See also autothrottle settings and docs
      30. 
		
    30. 
			#DOWNLOAD_DELAY = 3
      31. 
		
    31. 
			# The download delay setting will honor only one of:
      32. 
		
    32. 
			#CONCURRENT_REQUESTS_PER_DOMAIN = 16
      33. 
		
    33. 
			#CONCURRENT_REQUESTS_PER_IP = 16
      34. 
		
    34. 
			 
      35. 
		
    35. 
			# Disable cookies (enabled by default)
      36. 
		
    36. 
			#COOKIES_ENABLED = False
      37. 
		
    37. 
			 
      38. 
		
    38. 
			# Disable Telnet Console (enabled by default)
      39. 
		
    39. 
			#TELNETCONSOLE_ENABLED = False
      40. 
		
    40. 
			 
      41. 
		
    41. 
			# Override the default request headers:
      42. 
		
    42. 
			#DEFAULT_REQUEST_HEADERS = {
      43. 
		
    43. 
			# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
      44. 
		
    44. 
			# 'Accept-Language': 'en',
      45. 
		
    45. 
			#}
      46. 
		
    46. 
			 
      47. 
		
    47. 
			# Enable or disable spider middlewares
      48. 
		
    48. 
			# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
      49. 
		
    49. 
			#SPIDER_MIDDLEWARES = {
      50. 
		
    50. 
			# 'scrapyseleniumtest.middlewares.ScrapyseleniumtestSpiderMiddleware': 543,
      51. 
		
    51. 
			#}
      52. 
		
    52. 
			 
      53. 
		
    53. 
			# Enable or disable downloader middlewares
      54. 
		
    54. 
			# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
      55. 
		
    55. 
			#DOWNLOADER_MIDDLEWARES = {
      56. 
		
    56. 
			# 'scrapyseleniumtest.middlewares.ScrapyseleniumtestDownloaderMiddleware': 543,
      57. 
		
    57. 
			#}
      58. 
		
    58. 
			DOWNLOADER_MIDDLEWARES = {
      59. 
		
    59. 
			'scrapyseleniumtest.middlewares.SeleniumMiddleware': 543,
      60. 
		
    60. 
			}
      61. 
		
    61. 
			# Enable or disable extensions
      62. 
		
    62. 
			# See https://docs.scrapy.org/en/latest/topics/extensions.html
      63. 
		
    63. 
			#EXTENSIONS = {
      64. 
		
    64. 
			# 'scrapy.extensions.telnet.TelnetConsole': None,
      65. 
		
    65. 
			#}
      66. 
		
    66. 
			 
      67. 
		
    67. 
			# Configure item pipelines
      68. 
		
    68. 
			# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
      69. 
		
    69. 
			#ITEM_PIPELINES = {
      70. 
		
    70. 
			# 'scrapyseleniumtest.pipelines.ScrapyseleniumtestPipeline': 300,
      71. 
		
    71. 
			#}
      72. 
		
    72. 
			ITEM_PIPELINES = {
      73. 
		
    73. 
			'scrapyseleniumtest.pipelines.MongoPipeline': 300,
      74. 
		
    74. 
			}
      75. 
		
    75. 
			# Enable and configure the AutoThrottle extension (disabled by default)
      76. 
		
    76. 
			# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
      77. 
		
    77. 
			#AUTOTHROTTLE_ENABLED = True
      78. 
		
    78. 
			# The initial download delay
      79. 
		
    79. 
			#AUTOTHROTTLE_START_DELAY = 5
      80. 
		
    80. 
			# The maximum download delay to be set in case of high latencies
      81. 
		
    81. 
			#AUTOTHROTTLE_MAX_DELAY = 60
      82. 
		
    82. 
			# The average number of requests Scrapy should be sending in parallel to
      83. 
		
    83. 
			# each remote server
      84. 
		
    84. 
			#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
      85. 
		
    85. 
			# Enable showing throttling stats for every response received:
      86. 
		
    86. 
			#AUTOTHROTTLE_DEBUG = False
      87. 
		
    87. 
			 
      88. 
		
    88. 
			# Enable and configure HTTP caching (disabled by default)
      89. 
		
    89. 
			# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
      90. 
		
    90. 
			#HTTPCACHE_ENABLED = True
      91. 
		
    91. 
			#HTTPCACHE_EXPIRATION_SECS = 0
      92. 
		
    92. 
			#HTTPCACHE_DIR = 'httpcache'
      93. 
		
    93. 
			#HTTPCACHE_IGNORE_HTTP_CODES = []
      94. 
		
    94. 
			#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
      95. 
		
    95. 
			KEYWORDS=['iPad']
      96. 
		
    96. 
			MAX_PAGE=2
      97. 
		
    97. 
			 
      98. 
		
    98. 
			MONGO_URL = 'localhost
      



  
    
  
  
    
   
  
    
  
 
 
    
 
 
  
 
 
 
 

    




 



    


    相关教程