map.executor

# -*- coding:utf-8 -*-

"""
对某个目录下的文件求 md5 值
"""

import os
import hashlib
import time
from functools import reduce
from multiprocessing import Pool as ProcessPool
from multiprocessing.dummy import Pool as ThreadPool
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed


def traverse_dir(dir_='/home/hzcortex/Desktop'):
    for origin_dir, current_dir, filename in os.walk(dir_):
        for file_ in filename:

            yield origin_dir + file_


def create_hash(filename):
    hashlib.md5(filename)
    #c = reduce(lambda x, y: x * y, range(1, 100))
    time.sleep(0.001)


def fib(n):
    if n <= 2:
        return 1
    return fib(n - 1) + fib(n - 2)


if __name__ == '__main__':

    file_list = list(c.encode('utf-8') for c in traverse_dir())
    nums = list(range(20,40))
    # import random
    # random.shuffle(cc)
    print ('共有 {} 个文件'.format(len(file_list)))
    time1 = datetime.now()
    # for filename in file_list:
    #     create_hash(filename)
    # for num in nums:
    #     fib(num)
    # print ('普通执行状况: {}'.format((datetime.now() - time1).total_seconds()))

    pool = ProcessPool(4)
    time2 = datetime.now()
    #pool.map(create_hash, file_list)
    pool.map(fib, nums)
    print ('多进程运行状况: {}'.format((datetime.now() - time2).total_seconds()))

    pool = ThreadPool(4)
    time3 = datetime.now()
    #pool.map(create_hash, file_list)
    pool.map(fib, nums)
    print ('多线程运行状况: {}'.format((datetime.now() - time3).total_seconds()))

    time4 = datetime.now()
    with ThreadPoolExecutor(max_workers=4) as executor:
        executor.map(fib, nums)

    print ('ThreadPoolExecutor 运行状况: {}'.format(
        (datetime.now() - time4).total_seconds()))

    time5 = datetime.now()
    with ProcessPoolExecutor(max_workers=4) as executor:
        # executor.map(create_hash, file_list)
        executor.map(fib, nums)
    print ('ProcessPoolExcutor 运行状况: {}'.format(
        (datetime.now() - time5).total_seconds()))

    # 我要给小明来说一下这个问题：
    """
    普通执行状况: 12.71012
    多进程运行状况: 5.799454
    多线程运行状况: 18.085609
    ThreadPoolExecutor 运行状况: 18.403806
    ProcessPoolExcutor 运行状况: 5.764657

    http://stackoverflow.com/questions/18671528/processpoolexecutor-from-concurrent-futures-way-slower-than-multiprocessing-pool
    """