python multi-process pool example

python-multiprocess-example.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 
# computes CRC of each file in current working directory (including subdirectories)
# usage: python3 python-multiprocess-example.py
# 
import sys
import os
import glob
import zlib
from multiprocessing import Pool


def crc(fileName):
    prev = 0
    for eachLine in open(fileName,"rb"):
        prev = zlib.crc32(eachLine, prev)
    return "%X"%(prev & 0xFFFFFFFF)


def process_file(fn):
    if os.path.isfile(fn):
        file_crc = crc(fn)
        print('CRC of %s : %s' % (fn, file_crc))
        return file_crc
    else:
        print('%s is not a file' % fn)
        return ""


def main():
    cwd = os.path.realpath('.')
    files = []
    for name in glob.glob('%s/*' % cwd):
        files.append(name)

    print("There are %d files in the list" % len(files))

    agents = 10
    
    with Pool(processes=agents) as pool:
        result = pool.map(process_file, files)

    # [a1,a2,..] + [b1,b2,..] => [(a1,b1),(a2,b2),..]
    result = list(zip(files, result))

    print("Done : %s" % str(result))

if __name__ == '__main__':
    main()