utils.py

# coding=utf-8
import asyncio
import aiohttp
import requests
from requests.exceptions import ConnectionError


def parse_url(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0'
    }
    try:
        resp = requests.get(url, headers=headers)
        if resp.status_code == 200:
            print('ok')
            return resp.text
        return None
    except ConnectionError:
        print('Error.')
    return None

class Downloader(object):
    """
    python3.5的标准库自带的async和await指令，
    相当于3.5之前的 @asyncio.coroutine和yield from
    提供异步抓取
    """
    def __init__(self, urls):
        self.urls = urls
        self._htmls = []

    async def download_single_page(self, url):
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as resp:
                self._htmls.append(await resp.text())

    def download(self):
        loop = asyncio.get_event_loop()
        tasks = [self.download_single_page(url) for url in self.urls]
        loop.run_until_complete(asyncio.wait(tasks))

    @property
    def htmls(self):
        self.download()
        return self._htmls