爬虫(十七)：Scrapy框架(四) 对接selenium爬取京东商品数据(4)

当前位置:

首页 > Python基础教程 >

爬虫(十七)：Scrapy框架(四) 对接selenium爬取京东商品数据(4)

'div', class_="p-name p-name-type-2")

proc_dict['title'] = title.get_text().strip()

price = li.find(name='strong', class_="J_" + id)

proc_dict['price'] = price.get_text()

comment = li.find(name='a', id="J_comment_" + id)

proc_dict['comment'] = comment.get_text() + '条评论'

url = 'https://item.jd.com/' + id + '.html'

proc_dict['url'] = url

proc_dict['type'] = 'JINGDONG'

yield proc_dict

middlewares.py:

# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from urllib.parse import urlencode
from scrapy.http import HtmlResponse
from logging import getLogger
from selenium.common.exceptions import TimeoutException
import time
class ScrapyseleniumtestSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class SeleniumMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
def __init__(self,timeout=None):
self.logger=getLogger(__name__)
self.timeout = timeout
self.browser = webdriver.Chrome()
self.browser.

栏目列表