Keras - problem with session in multi-threaded environment.

main.py

from scipy import io
import numpy as np
from keras.utils import np_utils
from sklearn.model_selection import train_test_split
from PIL import Image
from sklearn.metrics import confusion_matrix
import tensorflow as tf
from threading import Thread

from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten
from keras.layers import Conv2D, MaxPool2D, Dropout
from keras.callbacks import Callback, TensorBoard
import keras.backend as K
import time


X = np.zeros((1000, 28, 28))
Y = np.zeros((1000))

resolution = 28  # 28x28 images, grayscale
classes = 10     # 10 letters: ABCDEFGHIJ

# transforming data for TensorFlow backend
X = np.transpose(X, (2, 0, 1))
X = X.reshape((-1, resolution, resolution, 1))
X = X.astype('float32') / 255.

# 3 -> [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.]
y = Y.astype('int32')
Y = np_utils.to_categorical(y, classes)

# splitting data into training and test sets
X_train, X_test, Y_train, Y_test = train_test_split(
    X, Y,
    test_size=0.20,
    random_state=137)

letters = "ABCDEFGHIJ"

def array_2d_to_image(array, autorescale=True):
    assert array.min() >= 0
    assert len(array.shape) == 2
    if array.max() <= 1 and autorescale:
        array = 255 * array
    array = array.astype('uint8')
    return Image.fromarray(array)




model = Sequential()

model.add(Conv2D(16, (3, 3), activation='relu',
                 input_shape=(resolution, resolution, 1), name='block1_conv1'))
model.add(Conv2D(16, (3, 3), activation='relu', name='block1_conv2')) 
model.add(MaxPool2D(name='block1_maxpool'))

model.add(Conv2D(32, (3, 3), activation='relu', name='block2_conv1')) 
model.add(Conv2D(32, (3, 3), activation='relu', name='block2_conv2')) 
model.add(MaxPool2D(name='block2_maxpool'))

model.add(Flatten(name='flatten'))
model.add(Dropout(0.5, name='dropout'))
model.add(Dense(classes, activation='softmax', name='output'))

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

graph = tf.get_default_graph()

def change_learning_rate(new_lr):
    time.sleep(5)

    # Need to explicitely store reference to graph from main thread and
    # set it here as default:
    #
    # with graph.as_default():
    #    K.set_value(model.optimizer.lr, float(new_lr))
    #
    
    # It will blow up, because set_value will try to create Assign Op using different graph
    K.set_value(model.optimizer.lr, float(new_lr))
    print 'Changed learning rate to {}'.format(new_lr)


# K.set_value lazily creates TF AssignOp and saves it for later. So if I would do this
#
# K.set_value(model.optimizer.lr, K.get_value(model.optimizer.lr))
#
# It would work, because other thread would use already created operation instead of trying to create new operation.

thread = Thread(target=change_learning_rate, args=(10,))
thread.start()

model.fit(X_train, Y_train,
          epochs=1000,
          batch_size=32,
          validation_data=(X_test, Y_test),
          verbose=2)