DIY LSTM in Keras

lstm.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Building an LSTM from scratch with Keras (if that's not a contradiction)"""

from keras import backend as K
from keras.engine.topology import Layer
import numpy as np
from keras.layers import RNN, Dense, Activation,LSTM
from keras.models import Model
import keras




class DiyLSTM(Layer):

    def __init__(self, units, **kwargs):
        self.units = units
        self.state_size = [units, units]
        super(DiyLSTM, self).__init__(**kwargs)

    def build(self, input_shape):
        self.w_xi = self.add_weight(name='w_xi',
                                    shape=(input_shape[-1], self.units), initializer='uniform')
        self.w_xf = self.add_weight(name='w_xf',
                                    shape=(input_shape[-1], self.units), initializer='uniform')
        self.w_xo = self.add_weight(name='w_xo',
                                    shape=(input_shape[-1], self.units), initializer='uniform')
        self.w_xc = self.add_weight(name='w_xc',
                                    shape=(input_shape[-1], self.units), initializer='uniform')
        self.w_hi = self.add_weight(name='w_hi',
                                    shape=(self.units, self.units), initializer='uniform')
        self.w_hf = self.add_weight(name='w_hf',
                                    shape=(self.units, self.units), initializer='uniform')
        self.w_ho = self.add_weight(name='w_ho',
                                    shape=(self.units, self.units), initializer='uniform')
        self.w_hc = self.add_weight(name='w_hc',
                                    shape=(self.units, self.units), initializer='uniform')
        self.b_i = self.add_weight(name='b_i',
                                    shape=(1, self.units), initializer='zeros')
        self.b_f = self.add_weight(name='b_f',
                                    shape=(1, self.units), initializer='zeros')
        self.b_o = self.add_weight(name='b_o',
                                    shape=(1, self.units), initializer='zeros')
        self.b_c = self.add_weight(name='b_c',
                                    shape=(1, self.units), initializer='zeros')

        self.built = True

    def call(self, x, states):
        h, c = states
        i = K.sigmoid(K.dot(x, self.w_xi) + K.dot(h, self.w_hi) + self.b_i)
        f = K.sigmoid(K.dot(x, self.w_xf) + K.dot(h, self.w_hf) + self.b_f)
        o = K.sigmoid(K.dot(x, self.w_xo) + K.dot(h, self.w_ho) + self.b_o)

        c_in = K.tanh(K.dot(x, self.w_xc) + K.dot(h, self.w_hc) + self.b_c)
        c_n = f * c + i * c_in
        h_n = o * K.tanh(c_n)

        return h_n, [h_n, c_n]

cell = DiyLSTM(256)
lstm_def = RNN(cell)
dense_def = Dense(128)
softmax_def = Activation('softmax')

maxlen = 40
inp = keras.Input((maxlen, 128))
lstm = lstm_def(inp)
dense = dense_def(lstm)
softmax = softmax_def(dense)
out = softmax

model = Model([inp], [out])

model.compile(optimizer=keras.optimizers.Adam(), loss='categorical_crossentropy')

text = """You have just found Keras.

Keras is a high-level neural networks API, written in Python and capable of running on top of TensorFlow, CNTK, or Theano. It was developed with a focus on enabling fast experimentation. Being able to go from idea to result with the least possible delay is key to doing good research.

Use Keras if you need a deep learning library that:

Allows for easy and fast prototyping (through user friendliness, modularity, and extensibility).
Supports both convolutional networks and recurrent networks, as well as combinations of the two.
Runs seamlessly on CPU and GPU.
Read the documentation at Keras.io.

Keras is compatible with: Python 2.7-3.6.

Guiding principles

User friendliness. Keras is an API designed for human beings, not machines. It puts user experience front and center. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear and actionable feedback upon user error.
Modularity. A model is understood as a sequence or a graph of standalone, fully-configurable modules that can be plugged together with as little restrictions as possible. In particular, neural layers, cost functions, optimizers, initialization schemes, activation functions, regularization schemes are all standalone modules that you can combine to create new models.
Easy extensibility. New modules are simple to add (as new classes and functions), and existing modules provide ample examples. To be able to easily create new modules allows for total expressiveness, making Keras suitable for advanced research.
Work with Python. No separate models configuration files in a declarative format. Models are described in Python code, which is compact, easier to debug, and allows for ease of extensibility.
Getting started: 30 seconds to Keras

The core data structure of Keras is a model, a way to organize layers. The simplest type of model is the Sequential model, a linear stack of layers. For more complex architectures, you should use the Keras functional API, which allows to build arbitrary graphs of layers.

Here is the Sequential model:

from keras.models import Sequential

model = Sequential()
Stacking layers is as easy as .add():

from keras.layers import Dense

model.add(Dense(units=64, activation='relu', input_dim=100))
model.add(Dense(units=10, activation='softmax'))
Once your model looks good, configure its learning process with .compile():

model.compile(loss='categorical_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])
If you need to, you can further configure your optimizer. A core principle of Keras is to make things reasonably simple, while allowing the user to be fully in control when they need to (the ultimate control being the easy extensibility of the source code).

model.compile(loss=keras.losses.categorical_crossentropy,
              optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True))
You can now iterate on your training data in batches:

# x_train and y_train are Numpy arrays --just like in the Scikit-Learn API.
model.fit(x_train, y_train, epochs=5, batch_size=32)
Alternatively, you can feed batches to your model manually:

model.train_on_batch(x_batch, y_batch)
Evaluate your performance in one line:

loss_and_metrics = model.evaluate(x_test, y_test, batch_size=128)
Or generate predictions on new data:

classes = model.predict(x_test, batch_size=128)
Building a question answering system, an image classification model, a Neural Turing Machine, or any other model is just as fast. The ideas behind deep learning are simple, so why should their implementation be painful?

For a more in-depth tutorial about Keras, you can check out:

Getting started with the Sequential model
Getting started with the functional API
In the examples folder of the repository, you will find more advanced models: question-answering with memory networks, text generation with stacked LSTMs, etc.

Installation

Before installing Keras, please install one of its backend engines: TensorFlow, Theano, or CNTK. We recommend the TensorFlow backend.

TensorFlow installation instructions.
Theano installation instructions.
CNTK installation instructions.
You may also consider installing the following optional dependencies:

cuDNN (recommended if you plan on running Keras on GPU).
HDF5 and h5py (required if you plan on saving Keras models to disk).
graphviz and pydot (used by visualization utilities to plot model graphs).
Then, you can install Keras itself. There are two ways to install Keras:

Install Keras from PyPI (recommended):
sudo pip install keras
If you are using a virtualenv, you may want to avoid using sudo:

pip install keras
Alternatively: install Keras from the Github source:
First, clone Keras using git:

git clone https://github.com/fchollet/keras.git
Then, cd to the Keras folder and run the install command:

cd keras
sudo python setup.py install
Switching from TensorFlow to CNTK or Theano

By default, Keras will use TensorFlow as its tensor manipulation library. Follow these instructions to configure the Keras backend.

Support

You can ask questions and join the development discussion:

On the Keras Google group.
On the Keras Slack channel. Use this link to request an invitation to the channel.
You can also post bug reports and feature requests (only) in Github issues. Make sure to read our guidelines first.

Why this name, Keras?

Keras (κέρας) means horn in Greek. It is a reference to a literary image from ancient Greek and Latin literature, first found in the Odyssey, where dream spirits (Oneiroi, singular Oneiros) are divided between those who deceive men with false visions, who arrive to Earth through a gate of ivory, and those who announce a future that will come to pass, who arrive through a gate of horn. It's a play on the words κέρας (horn) / κραίνω (fulfill), and ἐλέφας (ivory) / ἐλεφαίρομαι (deceive).

Keras was initially developed as part of the research effort of project ONEIROS (Open-ended Neuro-Electronic Intelligent Robot Operating System).

"Oneiroi are beyond our unravelling --who can be sure what tale they tell? Not all that men look for comes to pass. Two gates there are that give passage to fleeting Oneiroi; one is made of horn, one of ivory. The Oneiroi that pass through sawn ivory are deceitful, bearing a message that will not be fulfilled; those that come out through polished horn have truth behind them, to be accomplished for men who see them." Homer, Odyssey 19. 562 ff (Shewring translation).

Next 
Built with MkDocs using a theme provided by Read the Docs.
 GitHubNext »"""

text = ' '.join(text.split())


print(model.summary())

text_ints = [ord(x) for x in text if ord(x)<128]
onehot = keras.utils.to_categorical(text_ints, num_classes=128)
print(onehot.shape)

# break onehot into redundant chunks (technique stolen from https://github.com/fchollet/keras/blob/master/examples/lstm_text_generation.py)
step = 1

x = []
y = []
for i in range(0, len(text_ints) - 40, step):
    x.append(onehot[i: i + maxlen])
    y.append(onehot[i + maxlen])

x = np.array(x)
y = np.array(y)

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    """copied from https://github.com/fchollet/keras/blob/master/examples/lstm_text_generation.py"""
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

def generate_sentence(model, temperature=0.2, length=100):
    sentence = onehot[:maxlen].copy().reshape((1, maxlen, 128))
    input_seq = ''.join([chr(x) for x in text_ints[:maxlen]]) + '###'
    out_seq = ''

    for i in range(length):
        next_char = model.predict(sentence)[0]
        char = sample(next_char, temperature)
        out_seq += chr(char)
        sentence[0,:-1] = sentence[0,1:]
        sentence[0,-1,:] = 0
        sentence[0,-1, char] = 1

    return input_seq + out_seq #, [ord(x) for x in out_seq])


for i in range(1000000):
    history=model.fit(x, y, epochs=10)
    for diversity in [0.2, 0.5, 1.0, 1.2]:
        print(i, diversity, generate_sentence(model,diversity))
    model.save('lstm-from-scratch.h5')
    loss = history.history['loss'][-1]
    if loss < 0.05:
        model.save('lstm-from-scratch.h5')
        print(generate_sentence(model, 0.2, len(text_ints)))

hi,
please can you help me to applicate the lstm on my data