Remove all Bitwarden duplicates from json file

bitwarden_find_duplicates.py

import json
import os
from collections import defaultdict
import sys

def validate_json_structure(data):
    """Validate the basic structure of the Bitwarden JSON export"""
    if not isinstance(data, dict):
        raise ValueError("Invalid JSON format: Root must be an object")
    if 'items' not in data:
        raise ValueError("Invalid JSON format: Missing 'items' array")
    if not isinstance(data['items'], list):
        raise ValueError("Invalid JSON format: 'items' must be an array")

def get_uris_from_item(item):
    """Extract all URIs from an item"""
    if not item or not isinstance(item, dict):
        return set()
    
    login = item.get('login')
    if not login or not isinstance(login, dict):
        return set()
    
    uris = login.get('uris', []) or []
    if not isinstance(uris, list):
        return set()
    
    # Collect all URIs in a set
    uri_set = set()
    for uri_obj in uris:
        if isinstance(uri_obj, dict) and uri_obj.get('uri'):
            # Split multiple URIs if they are comma-separated
            for uri in uri_obj['uri'].split(','):
                uri_set.add(uri.strip())
    
    return uri_set

def items_are_duplicate(item1, item2):
    """Check whether two items are duplicates"""
    if not item1 or not item2:
        return False
        
    login1 = item1.get('login', {})
    login2 = item2.get('login', {})
    
    # Check username and password
    if login1.get('username') != login2.get('username'):
        return False
    if login1.get('password') != login2.get('password'):
        return False
    
    # Check if at least one URI matches
    uris1 = get_uris_from_item(item1)
    uris2 = get_uris_from_item(item2)
    
    return bool(uris1.intersection(uris2))

def get_user_confirmation():
    warning_text = """
⚠️  WARNING - USE AT YOUR OWN RISK ⚠️

This script will process your Bitwarden export file and:
1. Identify duplicates based on matching URIs, username, and password
2. Create two new files:
   - *_edited.json: Contains unique entries (first occurrence kept)
   - *_deleted.json: Contains all duplicate entries that were removed

To import the cleaned data into Bitwarden:
1. First, delete all items in your Bitwarden vault
2. Then import the '*_edited.json' file into Bitwarden

Are you sure you want to continue? (Yes/No): """
    
    while True:
        response = input(warning_text).strip().lower()
        if response == 'yes':
            return True
        elif response == 'no':
            return False
        else:
            print("\nPlease enter 'Yes' or 'No'")

def process_json_file(file_path):
    # Read and validate JSON file
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            data = json.load(f)
        validate_json_structure(data)
    except json.JSONDecodeError as e:
        raise ValueError(f"Invalid JSON file: {str(e)}")
    except Exception as e:
        raise ValueError(f"Error reading file: {str(e)}")
    
    # Copy all metadata
    output_data = {
        'encrypted': data.get('encrypted', False),
        'folders': data.get('folders', []),
        'collections': data.get('collections', []),
        'organizations': data.get('organizations', [])
    }
    
    items = data['items']
    duplicate_items = []
    unique_items = []
    used_indices = set()
    
    # Find duplicates
    for i in range(len(items)):
        if i in used_indices:
            continue
            
        item1 = items[i]
        is_duplicate = False
        
        # Compare with all subsequent items
        for j in range(i + 1, len(items)):
            if j in used_indices:
                continue
                
            item2 = items[j]
            if items_are_duplicate(item1, item2):
                if not is_duplicate:
                    unique_items.append(item1)
                    is_duplicate = True
                duplicate_items.append(item2)
                used_indices.add(j)
        
        if not is_duplicate:
            unique_items.append(item1)
        used_indices.add(i)
    
    # Get base filename without extension
    base_name = os.path.splitext(file_path)[0]
    
    # Create output data with duplicates
    deleted_data = output_data.copy()
    deleted_data['items'] = duplicate_items
    
    # Create output data with cleaned entries
    edited_data = output_data.copy()
    edited_data['items'] = unique_items
    
    # Write duplicates to _deleted file
    deleted_file = f"{base_name}_deleted.json"
    with open(deleted_file, 'w', encoding='utf-8') as f:
        json.dump(deleted_data, f, indent=2, ensure_ascii=False)
    
    # Write cleaned data to _edited file
    edited_file = f"{base_name}_edited.json"
    with open(edited_file, 'w', encoding='utf-8') as f:
        json.dump(edited_data, f, indent=2, ensure_ascii=False)
    
    return len(duplicate_items), len(unique_items)

def main():
    # Get file path from user input
    file_path = input("Please enter the path to your Bitwarden JSON export file: ")
    
    if not os.path.exists(file_path):
        print(f"\nError: File '{file_path}' does not exist!")
        sys.exit(1)
    
    # Get user confirmation
    if not get_user_confirmation():
        print("\nOperation cancelled by user.")
        sys.exit(0)
    
    try:
        duplicates, uniques = process_json_file(file_path)
        print(f"\n✅ Processing completed successfully!")
        print(f"📊 Statistics:")
        print(f"   - {uniques} unique entries saved to '*_edited.json'")
        print(f"   - {duplicates} duplicate entries saved to '*_deleted.json'")
        print("\n📝 Next steps:")
        print("1. Backup your Bitwarden vault")
        print("2. Delete all items in your vault")
        print("3. Import the '*_edited.json' file into Bitwarden")
    except Exception as e:
        print(f"\n❌ An error occurred: {str(e)}")
        sys.exit(1)

if __name__ == "__main__":
    main() 

Output:

1. File with all removed items
2. Cleaned file without duplicates

Just import the _edited.json file to your Bitwarden Vault Webinterface