Empty Variable Within Instance Of A Class, Despite Specifically Setting It

When I run the following code: import scrapy from scrapy.crawler import CrawlerProcess class QuotesSpider(scrapy.Spider): name = 'quotes' search_url = '' def start_re

Solution 1:

The neatest way to do this, would be to use the constructor __init__(), but an easier(maybe just faster for what you want) is to move the definition of start_url inside the class. For example:

import scrapy
from scrapy.crawler import CrawlerProcess

class QuotesSpider(scrapy.Spider):

    name = "quotes"
    search_url = 'http://quotes.toscrape.com/page/1/'

    def start_requests(self):
        print ('search_url is currently: ' + self.search_url)
        yield scrapy.Request(url=self.search_url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = 'quotes-%s.html' % page
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log('Saved file %s' % filename)

process = CrawlerProcess({
    'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
})

test_spider = QuotesSpider()

process.crawl(test_spider)
process.start()