cyrreconstruct.py

#!/usr/bin/env python3

# This utility helps reconstructing mailboxes that we recreated after a botched upgrade
# to cyrus 3.6

# This code automates the suggestion from ellie timoney here: https://cyrus.topicbox.com/groups/info/T9d294f89a3d1d260-M2f194782b7c5b5b01e200409
# and reconstructs mailboxes that were recreated (and the old content lost) during an update from cyrus 3.4 to cyrus 3.6
#
# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1007965 for the related Debian bugreport
#
# Prerequisites: check out `python_cyrus` from https://github.com/larsimmisch/python_cyrus (in the directory where you run this script)
#
# It needs to be run as the `cyrus` user
#
# Use it at your own risk!


import sys
import os
from argparse import ArgumentParser
from getpass import getpass
import subprocess
import shutil
from pathlib import Path
from python_cyrus.source import cyruslib

cyrus_maildir = '/var/spool/cyrus/mail'

def scan_directories(path, stack=None):
    it = os.scandir(path)
    for entry in it:
        if entry.is_dir():
            if stack is None:
                new_stack = [entry.name]
                yield new_stack
            else:
                new_stack = stack + [entry.name]
                yield new_stack
            yield from scan_directories(os.path.join(path, entry.name), new_stack)

# move all files from subfolder source in cyrus_maildir to subfolder dest except files that start with "cyrus."
# in default-mode when file already exists in dest, it is not moved and a warning is printed.
# with force=True files are overwritten even if they exist in dest. With dry_run=True nothing is moved, but warnings are printed.
def move_mailbox_files(source, dest, force=False, dry_run=False):

    source = os.path.join(cyrus_maildir, source)
    dest = os.path.join(cyrus_maildir, dest)

    with os.scandir(source) as dir:
        for entry in dir:
            if entry.is_file() and not entry.name.startswith('cyrus.'):
                path_dest = Path(dest, entry.name)
                dest_exists = path_dest.is_file()
                if dest_exists:
                    print(f'warning: {str(path_dest)} exists')

                if not dry_run and (force or not dest_exists):
                    shutil.move(entry, path_dest)

def old_mailbox_prefix(user):
    # old mailbox starts with first character of user name, e.g. l/user/lars for user='lars'
    first = user[0]
    return f'{first}/user/{user}'

def old_mailbox_path(user, paths):
    return os.path.join(old_mailbox_prefix(user), *paths)

def mailbox_basename(imap, user):
   return 'user' + imap.SEP + user

def mailbox_name(imap, user, paths=None):
    return mailbox_basename(imap, user) + imap.SEP + imap.SEP.join(paths)

if __name__ == '__main__':

    parser = ArgumentParser(description='reconstruct cyrus mailboxes after failed UUID migration')
    parser.add_argument('user', nargs='+', help='user name')
    parser.add_argument('-a', '--admin', type=str, default='cyrus', help='admin user')
    parser.add_argument('-d', '--dry-run', action='store_true', help="don't actually move files or create mailboxes")

    args = parser.parse_args()

    # connect to cyrus
    password = getpass(f'admin password for {args.admin}: ')
    imap = cyruslib.CYRUS("imaps://127.0.0.1:993")
    imap.login('cyrus', password)

    mailboxes = {}

    for user in args.user:
        # mailbox is a list of paths
        # the empty list [] is the basename
        mailboxes[user] = [[]] + [mailbox for mailbox in scan_directories(os.path.join(cyrus_maildir, old_mailbox_prefix(user)))]
        imap_user_basename = mailbox_basename(imap, user)

        existing = imap.lm(imap_user_basename + imap.SEP + '*')

        # create same format as the mailboxes dict (list of list with path components without base name)
        existing = [[]] + [m[len(imap_user_basename)+1:].split(imap.SEP) for m in existing if m.startswith(imap_user_basename)]

        print(mailboxes[user])

        for mb in mailboxes[user]:
            mailbox_name = imap.SEP.join([imap_user_basename] + mb)
            try:
                existing_mb = existing[existing.index(mb)]
            except ValueError:
                print(f'creating {mailbox_name}')
                if not args.dry_run:
                    # We need to quote the mailbox name in case it has spaces
                    # This should ideally be fixed in imaplib/python_cyrus instead
                    imap.cm(f'"{mailbox_name}"')

            try:
                real_path = subprocess.check_output(['/usr/lib/cyrus/bin/mbpath', mailbox_name], stderr=sys.stderr).decode('utf-8')
                real_path = real_path.strip()
            except subprocess.CalledProcessError:
                if args.dry_run:
                    real_path = '<path does not exist yet>'
                else:
                    raise

            print(f'moving {mailbox_name} to {real_path}')
            move_mailbox_files(old_mailbox_path(user, mb), real_path, dry_run=args.dry_run)

        print(f'reconstructing {imap_user_basename}')
        if not args.dry_run:
            subprocess.check_output(['/usr/lib/cyrus/bin/reconstruct', '-r', '-f', imap_user_basename], stderr=sys.stderr).decode('utf-8')

    imap.logout()

Thank you for providing this. It totally saved my life after a failed update to Debian Bookworm.
I've made slight modifications to ensure the restructuring actually works - updated version here:
https://gist.github.com/BastianMuc/4afdf1010f527dde982f556b6eab8664

Note: This only works if the affected user already has a mailbox which is found under cyradm. If not, just create one using cyradm:
cm user.testusername
... and this script will work like a charm.

Only thing I could not solve is to recover the prior read/seen statuses of the recovered messages. If someone has a hint how to do this, highly appreciated!

Thanks! I have updated the gist with your changes.

As for the read/seen status: I don't know how to recover them (my users certainly complained that they had to mark 30000+ messages as read)