Cleaning up duplicates in Dropbox's Camera Uploads

dbdupes.py

#!/usr/bin/env python
import sys, os, re
from PIL import Image, ImageChops

def find(root, pattern):
  for root, dns, fns in os.walk(root):
    for fn in fns:
      if re.match(pattern, fn):
        yield (root, fn)

def imgequal(im1, im2):
  return ImageChops.difference(im1, im2).getbbox() is None

def finddupes():
  dupe_re = r'^(?P<name>.*?)-1\.jpg$'

  for d, f in find('.', dupe_re):
    dupe_path = os.path.join(d, f)
    orig_path = os.path.join(d, re.match(dupe_re, f).groupdict()['name'] + '.jpg')

    if os.path.isfile(orig_path) and imgequal(Image.open(dupe_path), Image.open(orig_path)):
      yield dupe_path

if __name__ == '__main__':
  if len(sys.argv) == 1:
    for f in finddupes():
      sys.stdout.write(f)
      sys.stdout.write(os.linesep)
      sys.stdout.flush()
  else:
    print '''USAGE:

This tool finds duplicate Camera Uploads in your Dropbox. These are characterised by:
1) Having a name that looks like 2020.12.34 56.78.12-1.jpg (see the '-1' at the end)
2) Having exactly the same pixel content

This script searches from the current dir and calculates image difference (with PIL),
then outputs duplicate paths (the '-1' names) for which there is no difference with
the original (the file without '-1', IF it exists).

To delete duplicate images you can use this script like so:

$ ./dbdupes.py | xargs -t -I_ rm _

(I know, xargs is a piece of shit)

'''