Example of the non-backwards compatibility of pickle files

test_pickle_consume.py

import pickle


class MyClass:
    """Example class to pickle. Version 2."""

    def __init__(self, a, b, c, d=4):
        """New field `d` with default value, so no problems right?"""
        self.a = a
        self.b = b
        self.c = c
        self.d = d

    def __repr__(self):
        return f"{self.a=}, {self.b=}, {self.c=}, {self.d=}"


with open("my_data.pkl", "rb") as f:
    my_data = pickle.load(f)

print(my_data)

test_pickle_consume_fixed.py

import pickle


class MyClass:
    """Example class to pickle. Version 3. Now with backwards compatibility!"""

    def __init__(self, a, b, c, d=4):
        self.a = a
        self.b = b
        self.c = c
        self.d = d

    def __repr__(self):
        return f"{self.a=}, {self.b=}, {self.c=}, {self.d=}"

    def __setstate__(self, state):
        """Called when un-pickling. Make sure fields have default values."""
        self.a = state.get("a", 1)
        self.b = state.get("b", 2)
        self.c = state.get("c", 3)
        self.d = state.get("d", 4)


with open("my_data.pkl", "rb") as f:
    my_data = pickle.load(f)

print(my_data)

test_pickle_produce.py

import pickle


class MyClass:
    """Example class to pickle. Version 1."""

    def __init__(self, a, b, c):
        """No field `d` yet."""
        self.a = a
        self.b = b
        self.c = c

    def __repr__(self):
        return f"{self.a=}, {self.b=}, {self.c=}"


my_data = MyClass(1, 2, 3)
print(my_data)

with open("my_data.pkl", "wb") as f:
    pickle.dump(my_data, f)

First run test_pickle_produce.py, this will produce a pickled file my_data.pkl. Then try to run test_pickle_consume.py to load in the data. Only now the MyClass class has been updated with an additional field. Loading the data doesn't work anymore :(

The solution is to provide a default value for d in MyClass.__init__.

The solution is to write custom MyClass.__getstate__ and MyClass.__setstate__ methods. See test_pickle_consume_fixed.py for an example.