Detect identical files

detect_identical_files.py

import hashlib
from glob import glob
from pathlib import Path
import argparse
import os


def get_args():
    parser = argparse.ArgumentParser(
        "Detect identical files."
    )
    parser.add_argument("--path", type=str, help="absolute path to folder to analyze")
    parser.add_argument("--recursive", action='store_true', default=False, help="Detect indetical files recursively in subfolders")
    args = parser.parse_args()
    return args


def md5Checksum(filePath):
    with open(filePath, 'rb') as fh:
        m = hashlib.md5()
        while True:
            data = fh.read(8192)
            if not data:
                break
            m.update(data)
        return m.hexdigest()


if __name__ == "__main__":
    # Load parameters
    opt = get_args()

    path = opt.path # Put path of the data
    # Use or not recursive
    if opt.recursive:
        rec = '**/*'
    else:
        rec = '*'
    hashmap = {}
    for f in Path(path).glob(rec):
        if not os.path.isfile(f):
            continue
        has_ = md5Checksum(f)
        if has_ in hashmap:
            hashmap[has_].append(os.path.realpath(f))
        else:
            hashmap[has_] = [os.path.realpath(f)]

    for hash_,values in hashmap.items():
      if len(values)>1:
        print("files with hash ", hash_, " are:", values)