加速爬虫: 多进程分布式 | 爬取莫烦python的网页

import multiprocessing as mp
import time
from urllib.request import urlopen
from urllib.parse import urljoin
from bs4 import BeautifulSoup
import re

#base_url = "http://127.0.0.1:4000/"
base_url = 'https://morvanzhou.github.io/'

# DON'T OVER CRAWL THE WEBSITE OR YOU MAY NEVER VISIT AGAIN
if base_url != "http://127.0.0.1:4000/":
    restricted_crawl = True
else:
    restricted_crawl = False

def crawl(url):
    response = urlopen(url)
    time.sleep(0.1)             # slightly delay for downloading
    return response.read().decode()
def parse(html):
    soup = BeautifulSoup(html, 'lxml')
    urls = soup.find_all('a', {"href": re.compile('^/.+?/$')})
    title = soup.find('h1').get_text().strip()
    page_urls = set([urljoin(base_url, url['href']) for url in urls])
    url = soup.find('meta', {'property': "og:url"})['content']
    return title, page_urls, url

if __name__ == '__main__':
    unseen = set([base_url, ])
    seen = set()

    pool = mp.Pool(4)
    count, t1 = 1, time.time()
    while len(unseen) != 0:  # still get some url to visit
        if restricted_crawl and len(seen) > 20:
            break
        print('\nDistributed Crawling...')
        crawl_jobs = [pool.apply_async(crawl, [url]) for url in unseen]
        htmls = [j.get() for j in crawl_jobs]  # request connection

        print('\nDistributed Parsing...')
        parse_jobs = [pool.apply_async(parse, [html]) for html in htmls]
        results = [j.get() for j in parse_jobs]  # parse html

        print('\nAnalysing...')
        seen.update(unseen)  # seen the crawled
        unseen.clear()  # nothing unseen

        for title, page_urls, url in results:
            print(count, title, url)
            count += 1
            unseen.update(page_urls - seen)  # get new url to crawl
    print('Total time: %.1f s' % (time.time() - t1,))  # 16 s !!!

#     C:\Users\11140\Anaconda3\python.exe D:/crawler/main.py

# Distributed Crawling...

# Distributed Parsing...

# Analysing...
# 1 教程 https://morvanzhou.github.io/

# Distributed Crawling...

# Distributed Parsing...

# Analysing...
# 2 高级爬虫: 高效无忧的 Scrapy 爬虫库 https://morvanzhou.github.io/tutorials/data-manipulation/scraping/5-02-scrapy/
# 3 Numpy & Pandas 教程系列 https://morvanzhou.github.io/tutorials/data-manipulation/np-pd/
# 4 Git 版本管理 教程系列 https://morvanzhou.github.io/tutorials/others/git/
# 5 计算机视觉 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/computer-vision/
# 6 multiprocessing 多进程教程系列 https://morvanzhou.github.io/tutorials/python-basic/multiprocessing/
# 7 为了更优秀 https://morvanzhou.github.io/support/
# 8 Why? https://morvanzhou.github.io/tutorials/data-manipulation/scraping/1-00-why/
# 9 Theano 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/theano/
# 10 迁移学习 Transfer Learning https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/5-16-transfer-learning/
# 11 其他教学系列 https://morvanzhou.github.io/tutorials/others/
# 12 进化算法 Evolutionary Strategies 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/evolutionary-algorithm/
# 13 基础教程系列 https://morvanzhou.github.io/tutorials/python-basic/basic/
# 14 Keras 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/keras/
# 15 Python基础 教程系列 https://morvanzhou.github.io/tutorials/python-basic/
# 16 Threading 多线程教程系列 https://morvanzhou.github.io/tutorials/python-basic/threading/
# 17 高级爬虫: 让 Selenium 控制你的浏览器帮你爬 https://morvanzhou.github.io/tutorials/data-manipulation/scraping/5-01-selenium/
# 18 Tensorflow 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/
# 19 网页爬虫教程系列 https://morvanzhou.github.io/tutorials/data-manipulation/scraping/
# 20 迁移学习 Transfer Learning https://morvanzhou.github.io/tutorials/machine-learning/ML-intro/2-9-transfer-learning/
# 21 有趣的机器学习系列 https://morvanzhou.github.io/tutorials/machine-learning/ML-intro/
# 22 推荐学习顺序 https://morvanzhou.github.io/learning-steps/
# 23 Pytorch 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/torch/
# 24 说吧~ https://morvanzhou.github.io/discuss/
# 25 近期更新 https://morvanzhou.github.io/recent-posts/
# 26 Tkinter GUI 教程系列 https://morvanzhou.github.io/tutorials/python-basic/tkinter/
# 27 机器学习实践 https://morvanzhou.github.io/tutorials/machine-learning/ML-practice/
# 28 数据处理教程系列 https://morvanzhou.github.io/tutorials/data-manipulation/
# 29 Linux 简易教学 https://morvanzhou.github.io/tutorials/others/linux-basic/
# 30 强化学习 Reinforcement Learning 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/
# 31 Matplotlib 画图教程系列 https://morvanzhou.github.io/tutorials/data-manipulation/plt/
# 32 关于莫烦 https://morvanzhou.github.io/about/
# 33 Sklearn 通用机器学习 教程系列 https://morvanzhou.github.io/tutorials/machine-learning/sklearn/
# 34 机器学习系列 https://morvanzhou.github.io/tutorials/machine-learning/
# Total time: 100.6 s

# Process finished with exit code 0