deja-dup / duplicity ubuntu backup: emulate restore with exclude list, for example to bypass corrupted volumes

examples_current-files-list.txt

Local and Remote metadata are synchronized, no sync needed.
Last full backup date: Wed Dec 14 02:45:44 2016
No signature chain for the requested time.  Using oldest available chain, starting at time Wed Dec 14 02:45:44 2016.
Wed Nov 30 17:24:55 2016 .
Thu Sep 15 18:29:12 2016 home
Thu Dec  8 20:12:07 2016 home/user
Fri Dec 18 16:24:55 2015 home/user/foo
Fri Dec 18 16:24:55 2015 home/user/foo/bar
Mon Dec 21 12:08:56 2015 home/user/foo/bar/file1
Mon Dec 21 12:08:57 2015 home/user/foo/bar/file2
Fri Dec 18 16:24:55 2015 home/user/foo/bar/subdir
Fri Dec 18 16:24:55 2015 home/user/foo/bar/subdir/file
Fri Dec 18 16:24:55 2015 home/user/foo/bar/file3
Fri Dec 18 16:24:55 2015 home/user/foo/baz
Fri Dec 18 16:24:55 2015 home/user/foo/baz/file
Fri Dec 18 16:24:55 2015 home/user/foo/baz/subdir
Fri Dec 18 16:24:55 2015 home/user/foo/baz/subdir/file
Fri Dec 18 16:24:55 2015 home/user/fooo1
Fri Dec 18 16:24:55 2015 home/user/fooo2

examples_exclude-list.txt

home/user/fooo1
home/user/foo/bar

examples_output_generated-include-list.txt

home/user/fooo2
home/user/foo/baz

examples_output_generated-include-list.txt.log

Parsing exclude list: 'examples/exclude-list.txt'
Exclude Tree: {'': {}, 'home': {'user': {'fooo1': {}, 'foo': {'bar': {}}}}}
Parsing file list: 'examples/current-files-list.txt'
Parsed 15 file list lines
Generated 2 include lines

examples_output_generated-include-list.txt.verbose.log

Parsing exclude list: 'examples/exclude-list.txt'
Exclude Tree: {'': {}, 'home': {'user': {'fooo1': {}, 'foo': {'bar': {}}}}}
Parsing file list: 'examples/current-files-list.txt'
Include Path 'home': True exclusion-in-sub-tree
Include Path 'home/user': True exclusion-in-sub-tree
Include Path 'home/user/foo': True exclusion-in-sub-tree
Include Path 'home/user/foo/bar': False excluded-sub-tree
Include Path 'home/user/foo/bar/file1': False excluded-sub-tree
Include Path 'home/user/foo/bar/file2': False excluded-sub-tree
Include Path 'home/user/foo/bar/subdir': False excluded-sub-tree
Include Path 'home/user/foo/bar/subdir/file': False excluded-sub-tree
Include Path 'home/user/foo/bar/file3': False excluded-sub-tree
Include Path 'home/user/foo/baz': True included-path
Include Path 'home/user/foo/baz/file': True included-by-parent
Include Path 'home/user/foo/baz/subdir': True included-by-parent
Include Path 'home/user/foo/baz/subdir/file': True included-by-parent
Include Path 'home/user/fooo1': False excluded-sub-tree
Include Path 'home/user/fooo2': True included-path
Parsed 15 file list lines
Generated 2 include lines

generate-file-to-restore-list.py

#! /usr/bin/python
# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
#
# Generates a list of include paths from the global file list and an exclude list, to be used with `duplicity restore --file-to-restore`, to emulate `duplicity restore --exclude`
#
# Usage:
# ./generate-file-to-restore-list.py current-files-list.txt exclude-list.txt > include-list.txt

from __future__ import print_function
import sys


import argparse
parser = argparse.ArgumentParser()
parser.add_argument("file_list", help="File containing file list form current-file-lists")
parser.add_argument("exclude_list", help="File containing exclude list")
parser.add_argument("-v", "--verbose", help="Print verbose messages", action="store_true")
args = parser.parse_args()


def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

verbose = args.verbose
def vprint(*args, **kwargs):
    if verbose:
        eprint(*args, **kwargs)

class FileTree(dict):
    """FileTree using auto vivification."""
    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

    def addPath(self, path):
        self.getPath(path)

    def getPath(self, path):
        """Get (and add via auto vivification) path."""
        components = path.split('/', 1)
        child = self[components[0]]
        if len(components) == 1:
            return child
        else:
            # recurse
            return child.getPath(components[1])

    def includePath(self, path, exclude):
        """Add include path to self FileTree by generating minimal include Tree that does not contain the paths in exclude FileTree.
        Return (excluded, detail)"""

        components = path.split('/', 1)
        name = components[0]
        if name in exclude:
            # there is an exclusion at this level or down the tree
            excludeChild = exclude[name]
            if not excludeChild:
                # it's a leaf on the exclude FileTree: exclude this whole sub-tree
                excluded = True
                detail = 'excluded-sub-tree'
            else:
                if len(components) == 1:
                    # path-recursion ended: this path contains excluded sub-tree: don't include it directly; other entries may add non-excluded children later
                    excluded = False
                    detail = 'exclusion-in-sub-tree'
                else:
                    # recurse
                    excluded, detail = self[name].includePath(components[1], excludeChild)
        else:
            # this sub-tree is not excluded
            # stop path-recursion here: add this level to include this sub-tree
            self[name]
            excluded = False
            detail = 'included-path' if len(components) == 1 else 'included-by-parent'
        return excluded, detail

    def list(self, prefix = ''):
        if not self:
            # we are a leaf on the FileTree
            yield prefix
        else:
            # recurse
            for name in self:
                for line in self[name].list((prefix+'/' if prefix else '')+name):
                    yield line


exclude = FileTree()
eprint("Parsing exclude list: '%s'"%(args.exclude_list))
for line in open(args.exclude_list):
    # Example:
    # home/user/.config
    exclude.addPath(line.rstrip('\n'))

eprint("Exclude Tree: %s"%(exclude))

t = FileTree()
eprint("Parsing file list: '%s'"%(args.file_list))
with open(args.file_list) as f:
    skip = True
    count = 0
    for line in f:
        line = line.rstrip('\n')
        # extact path
        # Example line:
        # Mon Dec  3 12:52:34 2012 home/user/.emacs
        path = line[25:]

        # skip headers before file list; to adapt if first path line is not for '.'
        # Example header:
        # Local and Remote metadata are synchronized, no sync needed.
        # Last full backup date: Wed Dec 14 02:45:44 2016
        # No signature chain for the requested time.  Using oldest available chain, starting at time Wed Dec 14 02:45:44 2016.
        # Wed Nov 30 17:24:55 2016 .
        # Thu Dec  8 20:12:07 2016 home/user
        # Thu Dec  8 18:42:03 2016 home/user/.emacs
        if skip and len(line) > 25 and path == '.':
            skip = False
            continue
        if skip:
            continue
        count += 1

        excluded, detail = t.includePath(path, exclude)
        vprint("Include Path '%s': %s %s"%(path, not excluded, detail))

eprint("Parsed %d file list lines"%(count))

count = 0
for f in t.list():
    print(f)
    count += 1

eprint("Generated %d include lines"%(count))