Асинхронный парсер сайта, часть 2: добавляем конкурентность

from urllib.parse import urljoin, urlparse, urldefrag
import asyncio
import logging

import aiohttp
from bs4 import BeautifulSoup

logging.basicConfig(level=logging.INFO)


async def fetch_and_validate_page_contents(url, session):
    while True:
        try:
            async with session.get(url) as response:
                if response.content_type != "text/html":
                    return None
                if response.url.host != HOSTNAME:
                    return None
                return await response.text()
        except aiohttp.ClientConnectorError:
            logging.warning(f"Connection error for URL: {url}, retrying in 0.5 second...")
            await asyncio.sleep(0.5)


async def parse_page(url, session):
    text = await fetch_and_validate_page_contents(url, session)
    if text is None:
        return None, set()
    soup = BeautifulSoup(text, "lxml")
    title = soup.find("title")
    if title is not None:
        title = title.text
    all_internal_links = set()
    for a in soup.find_all("a"):
        link = a.get("href")
        if link is None:
            continue
        if link.startswith("#"):
            continue
        absolute_link = urljoin(url, link)
        if urlparse(absolute_link).hostname == HOSTNAME:
            all_internal_links.add(
                urldefrag(absolute_link)[0]
            )
    return title, all_internal_links


async def process_site(website, session):
    visited_urls = {}
    urls_to_visit = {website}
    while urls_to_visit:
        tasks = {}
        for url in urls_to_visit:
            current_url_to_parse = url
            visited_urls[current_url_to_parse] = None
            tasks[url] = parse_page(current_url_to_parse, session)
        urls_to_visit.clear()
        results = await asyncio.gather(
            *tasks.values()
        )
        for url, (title, internal_links) in zip(tasks.keys(), results):
            visited_urls[url] = title
            for link in internal_links:
                if link not in visited_urls:
                    urls_to_visit.add(link)
        logging.info(f"Crawled {len(visited_urls)} pages, next iteration is {len(urls_to_visit)} pages")
    return visited_urls


async def main():
    async with aiohttp.ClientSession() as session:
        return await process_site(WEBSITE, session)


if __name__ == "__main__":
    WEBSITE = "https://www.python.org"
    HOSTNAME = urlparse(WEBSITE).hostname
    titles = asyncio.run(main())
    print(len(titles))
    print(titles[WEBSITE])