爬虫(十八)：Scrapy框架(五) Scrapy通用爬虫(3)

当前位置:

replace(',"',".0")

else:

score=score

else:

score='NULL'

return score

def get_story(self,response):

story=response.xpath('//div[@class="book-intro"]/p/text()').extract()[0]

if len(story)>0:

story=story.strip()

else:

story='NULL'

return story

def get_news(self,response):

news=response.xpath('//div[@class="detail"]/p[@class="cf"]/a/text()').extract()[0]

if len(news)>0:

news=news.strip()

else:

news='NULL'

return news

其他部分就没什么变化了，就settings加入了请求头：

scrapy crawl read

运行结果：

read.py:

# -*- coding: utf-8 -*-
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from qd.items import QdItem
import requests
class ReadSpider(CrawlSpider):
name = 'read'
# allowed_domains = ['qidian.com']
start_urls = ['https://www.qidian.com/all?orderId=&style=1&pageSize=20&siteid=1&pubflag=0&hiddenField=0&page=1']
rules = (
#匹配全部主页面的url规则深度爬取子页面
Rule(LinkExtractor(allow=(r'https://www.qidian.com/all\?orderId=\&style=1\&pageSize=20\&siteid=1\&pubflag=0\&hiddenField=0\&page=(\d+)')),follow=True),
#匹配详情页面不作深度爬取
Rule(LinkExtractor(allow=r'https://book.qidian.com/info/(\d+)'), callback='parse_item', follow=False),
)
def parse_item(self, response):
item=QdItem()
item['book_name']=self.get_book_name(response)
item['author']=self.get_author(response)
item['state']=self.get_state(response)
item['type']=self.get_type(response)
item['about']=self.get_about(response)
item['score']=self.get_score(response)
item['story']=self.get_story(response)
item['news']=self.get_news(response)
yield item
def get_book_name(self,response):
book_name=response.xpath('//h1/em/text()').extract()[0]
if len(book_name)>0:
book_name=book_name.strip()
else:
book_name='NULL'
return book_name
def get_author(self,response):
author=response.xpath('//h1/span/a/text()').extract()[0]
if len(author)>0:
author=author.strip()
else:
author='

栏目列表