diff --git a/hepcrawl/spiders/hindawi_spider.py b/hepcrawl/spiders/hindawi_spider.py index 6c2de27..ecc1a04 100644 --- a/hepcrawl/spiders/hindawi_spider.py +++ b/hepcrawl/spiders/hindawi_spider.py @@ -71,7 +71,10 @@ def __init__(self, source_file=None, *args, **kwargs): def start_requests(self): """Default starting point for scraping shall be the local XML file.""" self.log('Harvest started.', logging.INFO) - yield Request(self.source_file) + headers = { + 'User-Agent': 'curl/7.64.1' + } + yield Request(self.source_file, headers=headers) def parse_node(self, response, node): self.log('Parsing node...', logging.INFO)